fgg/fgg.rb

#!/bin/env ruby

# Copyright (c) 2022 Johannes Findeisen
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is furnished
# to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# fgg - A gallery generator for static site generators using Flickr as data source.
#
# THIS IS THE PROTOTYPE FOR THE PYTHON VERSION OF THIS SCRIPT! IT EXISTS BECAUSE I AM
# USING JEKYLL SINCE SOME YEARS FOR SITE GENERATION. SINCE I AM SWITCHING TO NIKOLA
# WHICH IS WRITTEN IN PYTHON THIS WILL BECOME OBSOLETE ASAP SINCE MY FAVORITE LANGUAGE
# IS PYTHON AND RUBY REALLY SUCKS IN MANY WAYS...
#
# This is a port from a prototype I have running since some years. I want to port
# this to Python in the future.
#
# Flickr API documentation
# https://www.flickr.com/services/api/

require 'date'
require 'digest'
require 'digest/crc32'
require 'fileutils'
require 'flickraw'
require 'json'
require 'openssl'

FlickRaw.api_key = ENV["FLICKR_API_KEY"]
FlickRaw.shared_secret = ENV["FLICKR_SHARED_SECRET"]
flickr.access_token = ENV["FLICKR_ACCESS_TOKEN"]
flickr.access_secret = ENV["FLICKR_ACCESS_SECRET"]

# error = 0
# warning = 1
# info = 2
# debug = 3
def barf(message, log)
    if log == 0
        puts("ERROR: " + message)
    end
    if log == 1
        puts("WARNING: " + message)
    end
    if log == 2
        puts("INFO: " + message)
    end
    if log == 3
        puts("DEBUG: " + message)
    end
end

def fill_zeros(number, digit_count=3)
    format_string = "%0#{digit_count}d"
    return format_string % (number % 10**digit_count)
end

def get_count(flickr, photos_per_page, page)
    # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html
    count = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page).length
end

barf("Logging in to Flickr...", 2)
login = flickr.test.login
barf("You are now authenticated as #{login.username}", 2)

# the timezone difference to UTC
tz_diff = "+01:00"

page = 1
page_count = 0
photo_count = 0
photos_per_page = 500
barf("Counting Pages...", 2)
while get_count(flickr, photos_per_page, page) != 0 do
    count = get_count(flickr, photos_per_page, page)
    photo_count = photo_count + count
        print "."
        STDOUT.flush
    page = page + 1
    page_count = page_count + 1
end
barf("", 2)

barf("Page Count: " + page_count.to_s(), 2)
barf("Photo Count: " + photo_count.to_s(), 2)

overall_tags = []
page = 1
photo = photo_count
photo_map = {}
while page <= page_count do
    barf("Page: " + page.to_s, 2)

    # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html
    photo_list = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page)
    photo_list.each_with_index do
        |flickr_photo|
        barf("Photo: " + photo.to_s, 2)

        ##puts(flickr_photo.inspect)
        ##exit

        if File.file?("cache/flickraw/" + flickr_photo.id + "/meta")
            flickr_photo_info = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/meta"))
        else
            # https://www.flickr.com/services/api/flickr.photos.getInfo.html
            flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id)
            FileUtils.mkdir_p("cache/flickraw/" + flickr_photo.id)
            File.open("cache/flickraw/" + flickr_photo.id + "/meta", 'wb') {|f| f.write(Marshal.dump(flickr_photo_info))}
        end
        #flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id)
        #puts(flickr_photo_info.inspect)
        #exit

        photo_timestamp = DateTime.parse(flickr_photo_info.dates.taken.to_s + tz_diff).to_time.to_i

        photo_suffix = fill_zeros(photo, 7)
        photo_cache_dir = "cache/photos/" + photo_timestamp.to_s + "-" + photo_suffix
        if Dir.exists?(photo_cache_dir)
            barf("File exists: " + photo_cache_dir, 1)
        else
            FileUtils.mkdir_p(photo_cache_dir)
        end

        barf("Processing photo number: " + photo.to_s, 2)

        barf("Flickr photo id: " + flickr_photo.id, 2)
        photo_map = photo_map.merge!("photo_flickr_id" => flickr_photo.id)

        barf("Flickr photo owner: " + flickr_photo.owner, 2)
        photo_map = photo_map.merge!("photo_flickr_owner" => flickr_photo.owner)

        barf("Flickr photo title: " + flickr_photo.title, 2)
        photo_map = photo_map.merge!("photo_flickr_title" => flickr_photo.title)

        ##barf("Flickr photo description: " + flickr_photo_info.description.partition("[").first, 2)
        ##photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description.partition("[").first)
        barf("Flickr photo description: " + flickr_photo_info.description, 2)
        photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description)

        barf("Flickr photo location: " + flickr_photo_info.owner["location"], 2)
        photo_map = photo_map.merge!("photo_flickr_location" => flickr_photo_info.owner["location"])

        barf("Flickr photo username: " + flickr_photo_info.owner["username"], 2)
        photo_map = photo_map.merge!("photo_flickr_username" => flickr_photo_info.owner["username"])

        barf("Date taken: " + flickr_photo_info.dates.taken.to_s, 2)
        photo_map = photo_map.merge!("photo_flickr_date_taken" => "" + flickr_photo_info.dates.taken.to_s + "")

        barf("Date taken timestamp: " + photo_timestamp.to_s, 2)
        photo_map = photo_map.merge!("photo_flickr_date_taken_timestamp" => photo_timestamp)

        barf("Date uploaded: " + flickr_photo_info.dateuploaded.to_s, 2)
        photo_map = photo_map.merge!("photo_flickr_date_uploaded_timestamp" => flickr_photo_info.dateuploaded.to_i)

        ##puts flickr_photo_info.dates.inspect
        ##puts flickr_photo_info.inspect

        # START ########################################################################################################
        # NEVER CHANGE THE FOLLOWING LINES (UNTIL "# END") EXPECT YOU REALLY NEED OR WANT TO CHANGE THE PHOTO ID!!!
        # ALL LINKS TO THIS PHOTO WILL BECOME BROKEN!!! - hanez
        # https://docs.ruby-lang.org/en/master/OpenSSL/Digest.html
        digest1 = OpenSSL::Digest.new("sha256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
        digest2 = OpenSSL::Digest.new("sha512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
        digest3 = OpenSSL::Digest.new("sha3-256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
        digest4 = OpenSSL::Digest.new("sha3-512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
        photo_id = Digest::CRC32c.hexdigest(digest1.to_s + digest2.to_s + digest3.to_s + digest4.to_s)
        barf("Photo id: " + photo_id, 2)
        photo_map = photo_map.merge!("photo_id" => photo_id)
        # END ##########################################################################################################

        FileUtils.mkdir_p "photo/" + photo_id

        ## https://westonganger.com/posts/get-substring-between-two-strings-or-characters-in-ruby
        ## get string between last "[" and last "]":
        ##tags = flickr_photo_info.description[/.*\[(.*)\]/, 1]
        ##if tags
        ##    barf("Flickr photo tags: " + tags, 2)
        ##    photo_map = photo_map.merge! "photo_flickr_tags" => tags
        ##else
        ##    barf("Flickr photo tags: " + "", 2)
        ##    photo_map = photo_map.merge! "photo_flickr_tags" => ""
        ##end
        ##puts(flickr_photo_info.tags.inspect)
        if File.file?(photo_cache_dir + "/tags.array")
            tags = Marshal.load(File.binread(photo_cache_dir + "/tags.array"))
        else
            tags = []
            flickr_photo_info.tags.each_with_index do
                |flickr_photo_tags|
                tags.push(flickr_photo_tags.raw.downcase)
            end
            File.open(photo_cache_dir + "/tags.array", 'wb') {|f| f.write(Marshal.dump(tags))}
        end
        overall_tags = overall_tags | tags
        photo_map = photo_map.merge!("photo_flickr_tags" => tags)

        # convert 2004-11-19 12:51:19 to 20041119125119
        photo_date = flickr_photo_info.dates.taken
        photo_date = photo_date.to_s()
        photo_date["-"] = ""
        photo_date["-"] = ""
        photo_date[" "] = ""
        photo_date[":"] = ""
        photo_date[":"] = ""
        barf("Photo date: " + photo_date, 2)
        photo_map = photo_map.merge!("photo_date" => photo_date)

        photo_map = photo_map.merge!("photo_timezone" => "UTC" + tz_diff)

        # the filename should be optimized to create a more beautiful filename
        filename = String.new(flickr_photo_info.title.tr("\s", "-"))
        filename = filename.delete("\'\"\.\s\,\!\/\:\(\)\[\]\;\?\&\<\>\$\%")
        filename = filename.gsub("--", "-")

        barf("Filename: " + filename, 2)
        photo_map = photo_map.merge!("photo_filename" => filename)

        if File.file?("cache/flickraw/" + flickr_photo.id + "/sizes")
            sizes = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/sizes"))
        else
            # https://www.flickr.com/services/api/flickr.photos.getSizes.html
            sizes = flickr.photos.getSizes(:photo_id => flickr_photo.id)
            File.open("cache/flickraw/" + flickr_photo.id + "/sizes", 'wb') {|f| f.write(Marshal.dump(sizes))}
        end

        #puts(sizes.inspect)
        #exit

        photo_sizes_map = {}
        sizes.each_with_index do
            |flickr_photo_sizes|

            photo_size_map = {}
            size = flickr_photo_sizes.url[-2, 1]
            if flickr_photo_sizes.label == "Square"
                size = "x"
            end
            barf("Photo size: " + size, 3)
            photo_size_map = photo_size_map.merge!("size" => size)

            photo_target_size = flickr_photo_sizes.label.tr(" ", "-")
            photo_target_file = "photo/" + photo_id + "/" + filename + "-" + photo_date + "-" + photo_suffix + "-" + photo_target_size + ".jpg"
            if !File.file?(photo_target_file)
                cmd = "wget -U Foo -O " + photo_target_file + " " + flickr_photo_sizes.source
                exec = `#{cmd}`
            end

            barf("Photo target: " + photo_target_file, 3)
            photo_size_map = photo_size_map.merge!("target_file" => photo_target_file)

            photo_digest = Digest::SHA1.file(photo_target_file)
            photo_digest = photo_digest.hexdigest
            photo_digest = photo_digest.to_s()
            barf("Photo digest (SHA1): " + photo_digest, 3)
            photo_size_map = photo_size_map.merge!("digest_sha1" => photo_digest)

            photo_digest = Digest::SHA256.file(photo_target_file)
            photo_digest = photo_digest.hexdigest
            photo_digest = photo_digest.to_s()
            barf("Photo digest (SHA256): " + photo_digest, 3)
            photo_size_map = photo_size_map.merge!("digest_sha256" => photo_digest)

            photo_digest = Digest::SHA512.file(photo_target_file)
            photo_digest = photo_digest.hexdigest
            photo_digest = photo_digest.to_s()
            barf("Photo digest (SHA512): " + photo_digest, 3)
            photo_size_map = photo_size_map.merge!("digest_sha512" => photo_digest)

            barf("Photo size label: " + flickr_photo_sizes.label, 3)
            photo_size_map = photo_size_map.merge!("label" => flickr_photo_sizes.label)

            barf("Photo size width: " + flickr_photo_sizes.width.to_s, 3)
            photo_size_map = photo_size_map.merge!("width" => flickr_photo_sizes.width)

            barf("Photo size height: " + flickr_photo_sizes.height.to_s, 3)
            photo_size_map = photo_size_map.merge!("height" => flickr_photo_sizes.height)

            photo_sizes_map = photo_sizes_map.merge!(size => photo_size_map)

            ##sleep 1
        end
        photo_map = photo_map.merge!("photo_sizes" => photo_sizes_map)

        photo = photo - 1

        #barf(photo_map.to_json, 3)
        #barf(JSON.pretty_generate(photo_map), 3)
        File.open(photo_cache_dir + "/meta.json", 'wb') {|f| f.write(JSON.pretty_generate(photo_map))}
        ##exit
        ##sleep 1
    end

    page = page + 1
    ##sleep 1
end

##puts(overall_tags.sort)
##puts(JSON.pretty_generate(overall_tags.sort))

barf("Finished...!", 2)