diff --git a/.gitignore b/.gitignore index 532df06..baeacc1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cache/* photo/* photos/* tmp/*.dump.json +tmp/*.rb venv bin/fgg diff --git a/tmp/fgg.rb b/tmp/fgg.rb deleted file mode 100644 index 701b4a9..0000000 --- a/tmp/fgg.rb +++ /dev/null @@ -1,317 +0,0 @@ -#!/bin/env ruby - -# Copyright (c) 2022 Johannes Findeisen -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is furnished -# to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF -# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# fgg - A gallery generator for static site generators using Flickr as data source. -# -# THIS IS THE PROTOTYPE FOR THE PYTHON VERSION OF THIS SCRIPT! IT EXISTS BECAUSE I AM -# USING JEKYLL SINCE SOME YEARS FOR SITE GENERATION. SINCE I AM SWITCHING TO NIKOLA -# WHICH IS WRITTEN IN PYTHON THIS WILL BECOME OBSOLETE ASAP SINCE MY FAVORITE LANGUAGE -# IS PYTHON AND RUBY REALLY SUCKS IN MANY WAYS... -# -# This is a port from a prototype I have running since some years. I want to port -# this to Python in the future. -# -# Flickr API documentation -# https://www.flickr.com/services/api/ - -require 'date' -require 'digest' -require 'digest/crc32' -require 'fileutils' -require 'flickraw' -require 'json' -require 'openssl' - -FlickRaw.api_key = ENV["FLICKR_API_KEY"] -FlickRaw.shared_secret = ENV["FLICKR_SHARED_SECRET"] -flickr.access_token = ENV["FLICKR_ACCESS_TOKEN"] -flickr.access_secret = ENV["FLICKR_ACCESS_SECRET"] - -# error = 0 -# warning = 1 -# info = 2 -# debug = 3 -def barf(message, log) - if log == 0 - puts("ERROR: " + message) - end - if log == 1 - puts("WARNING: " + message) - end - if log == 2 - puts("INFO: " + message) - end - if log == 3 - puts("DEBUG: " + message) - end -end - -def fill_zeros(number, digit_count=3) - format_string = "%0#{digit_count}d" - return format_string % (number % 10**digit_count) -end - -def get_count(flickr, photos_per_page, page) - # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html - count = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page).length -end - -barf("Logging in to Flickr...", 2) -login = flickr.test.login -barf("You are now authenticated as #{login.username}", 2) - -# the timezone difference to UTC -tz_diff = "+01:00" - -page = 1 -page_count = 0 -photo_count = 0 -photos_per_page = 500 -barf("Counting Pages...", 2) -while get_count(flickr, photos_per_page, page) != 0 do - count = get_count(flickr, photos_per_page, page) - photo_count = photo_count + count - print "." - STDOUT.flush - page = page + 1 - page_count = page_count + 1 -end -barf("", 2) - -barf("Page Count: " + page_count.to_s(), 2) -barf("Photo Count: " + photo_count.to_s(), 2) - -overall_tags = [] -page = 1 -photo = photo_count -photo_map = {} -while page <= page_count do - barf("Page: " + page.to_s, 2) - - # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html - photo_list = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page) - photo_list.each_with_index do - |flickr_photo| - barf("Photo: " + photo.to_s, 2) - - ##puts(flickr_photo.inspect) - ##exit - - if File.file?("cache/flickraw/" + flickr_photo.id + "/meta") - flickr_photo_info = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/meta")) - else - # https://www.flickr.com/services/api/flickr.photos.getInfo.html - flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id) - FileUtils.mkdir_p("cache/flickraw/" + flickr_photo.id) - File.open("cache/flickraw/" + flickr_photo.id + "/meta", 'wb') {|f| f.write(Marshal.dump(flickr_photo_info))} - end - #flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id) - #puts(flickr_photo_info.inspect) - #exit - - photo_timestamp = DateTime.parse(flickr_photo_info.dates.taken.to_s + tz_diff).to_time.to_i - - photo_suffix = fill_zeros(photo, 7) - photo_cache_dir = "cache/photos/" + photo_timestamp.to_s + "-" + photo_suffix - if Dir.exists?(photo_cache_dir) - barf("File exists: " + photo_cache_dir, 1) - else - FileUtils.mkdir_p(photo_cache_dir) - end - - barf("Processing photo number: " + photo.to_s, 2) - - barf("Flickr photo id: " + flickr_photo.id, 2) - photo_map = photo_map.merge!("photo_flickr_id" => flickr_photo.id) - - barf("Flickr photo owner: " + flickr_photo.owner, 2) - photo_map = photo_map.merge!("photo_flickr_owner" => flickr_photo.owner) - - barf("Flickr photo title: " + flickr_photo.title, 2) - photo_map = photo_map.merge!("photo_flickr_title" => flickr_photo.title) - - ##barf("Flickr photo description: " + flickr_photo_info.description.partition("[").first, 2) - ##photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description.partition("[").first) - barf("Flickr photo description: " + flickr_photo_info.description, 2) - photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description) - - barf("Flickr photo location: " + flickr_photo_info.owner["location"], 2) - photo_map = photo_map.merge!("photo_flickr_location" => flickr_photo_info.owner["location"]) - - barf("Flickr photo username: " + flickr_photo_info.owner["username"], 2) - photo_map = photo_map.merge!("photo_flickr_username" => flickr_photo_info.owner["username"]) - - barf("Date taken: " + flickr_photo_info.dates.taken.to_s, 2) - photo_map = photo_map.merge!("photo_flickr_date_taken" => "" + flickr_photo_info.dates.taken.to_s + "") - - barf("Date taken timestamp: " + photo_timestamp.to_s, 2) - photo_map = photo_map.merge!("photo_flickr_date_taken_timestamp" => photo_timestamp) - - barf("Date uploaded: " + flickr_photo_info.dateuploaded.to_s, 2) - photo_map = photo_map.merge!("photo_flickr_date_uploaded_timestamp" => flickr_photo_info.dateuploaded.to_i) - - ##puts flickr_photo_info.dates.inspect - ##puts flickr_photo_info.inspect - - # START ######################################################################################################## - # NEVER CHANGE THE FOLLOWING LINES (UNTIL "# END") EXPECT YOU REALLY NEED OR WANT TO CHANGE THE PHOTO ID!!! - # ALL LINKS TO THIS PHOTO WILL BECOME BROKEN!!! - hanez - # https://docs.ruby-lang.org/en/master/OpenSSL/Digest.html - digest1 = OpenSSL::Digest.new("sha256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s) - digest2 = OpenSSL::Digest.new("sha512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s) - digest3 = OpenSSL::Digest.new("sha3-256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s) - digest4 = OpenSSL::Digest.new("sha3-512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s) - photo_id = Digest::CRC32c.hexdigest(digest1.to_s + digest2.to_s + digest3.to_s + digest4.to_s) - barf("Photo id: " + photo_id, 2) - photo_map = photo_map.merge!("photo_id" => photo_id) - # END ########################################################################################################## - - FileUtils.mkdir_p "photo/" + photo_id - - ## https://westonganger.com/posts/get-substring-between-two-strings-or-characters-in-ruby - ## get string between last "[" and last "]": - ##tags = flickr_photo_info.description[/.*\[(.*)\]/, 1] - ##if tags - ## barf("Flickr photo tags: " + tags, 2) - ## photo_map = photo_map.merge! "photo_flickr_tags" => tags - ##else - ## barf("Flickr photo tags: " + "", 2) - ## photo_map = photo_map.merge! "photo_flickr_tags" => "" - ##end - ##puts(flickr_photo_info.tags.inspect) - if File.file?(photo_cache_dir + "/tags.array") - tags = Marshal.load(File.binread(photo_cache_dir + "/tags.array")) - else - tags = [] - flickr_photo_info.tags.each_with_index do - |flickr_photo_tags| - tags.push(flickr_photo_tags.raw.downcase) - end - File.open(photo_cache_dir + "/tags.array", 'wb') {|f| f.write(Marshal.dump(tags))} - end - overall_tags = overall_tags | tags - photo_map = photo_map.merge!("photo_flickr_tags" => tags) - - # convert 2004-11-19 12:51:19 to 20041119125119 - photo_date = flickr_photo_info.dates.taken - photo_date = photo_date.to_s() - photo_date["-"] = "" - photo_date["-"] = "" - photo_date[" "] = "" - photo_date[":"] = "" - photo_date[":"] = "" - barf("Photo date: " + photo_date, 2) - photo_map = photo_map.merge!("photo_date" => photo_date) - - photo_map = photo_map.merge!("photo_timezone" => "UTC" + tz_diff) - - # the filename should be optimized to create a more beautiful filename - filename = String.new(flickr_photo_info.title.tr("\s", "-")) - filename = filename.delete("\'\"\.\s\,\!\/\:\(\)\[\]\;\?\&\<\>\$\%") - filename = filename.gsub("--", "-") - - barf("Filename: " + filename, 2) - photo_map = photo_map.merge!("photo_filename" => filename) - - if File.file?("cache/flickraw/" + flickr_photo.id + "/sizes") - sizes = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/sizes")) - else - # https://www.flickr.com/services/api/flickr.photos.getSizes.html - sizes = flickr.photos.getSizes(:photo_id => flickr_photo.id) - File.open("cache/flickraw/" + flickr_photo.id + "/sizes", 'wb') {|f| f.write(Marshal.dump(sizes))} - end - - #puts(sizes.inspect) - #exit - - photo_sizes_map = {} - sizes.each_with_index do - |flickr_photo_sizes| - - photo_size_map = {} - size = flickr_photo_sizes.url[-2, 1] - if flickr_photo_sizes.label == "Square" - size = "x" - end - barf("Photo size: " + size, 3) - photo_size_map = photo_size_map.merge!("size" => size) - - photo_target_size = flickr_photo_sizes.label.tr(" ", "-") - photo_target_file = "photo/" + photo_id + "/" + filename + "-" + photo_date + "-" + photo_suffix + "-" + photo_target_size + ".jpg" - if !File.file?(photo_target_file) - cmd = "wget -U Foo -O " + photo_target_file + " " + flickr_photo_sizes.source - exec = `#{cmd}` - end - - barf("Photo target: " + photo_target_file, 3) - photo_size_map = photo_size_map.merge!("target_file" => photo_target_file) - - photo_digest = Digest::SHA1.file(photo_target_file) - photo_digest = photo_digest.hexdigest - photo_digest = photo_digest.to_s() - barf("Photo digest (SHA1): " + photo_digest, 3) - photo_size_map = photo_size_map.merge!("digest_sha1" => photo_digest) - - photo_digest = Digest::SHA256.file(photo_target_file) - photo_digest = photo_digest.hexdigest - photo_digest = photo_digest.to_s() - barf("Photo digest (SHA256): " + photo_digest, 3) - photo_size_map = photo_size_map.merge!("digest_sha256" => photo_digest) - - photo_digest = Digest::SHA512.file(photo_target_file) - photo_digest = photo_digest.hexdigest - photo_digest = photo_digest.to_s() - barf("Photo digest (SHA512): " + photo_digest, 3) - photo_size_map = photo_size_map.merge!("digest_sha512" => photo_digest) - - barf("Photo size label: " + flickr_photo_sizes.label, 3) - photo_size_map = photo_size_map.merge!("label" => flickr_photo_sizes.label) - - barf("Photo size width: " + flickr_photo_sizes.width.to_s, 3) - photo_size_map = photo_size_map.merge!("width" => flickr_photo_sizes.width) - - barf("Photo size height: " + flickr_photo_sizes.height.to_s, 3) - photo_size_map = photo_size_map.merge!("height" => flickr_photo_sizes.height) - - photo_sizes_map = photo_sizes_map.merge!(size => photo_size_map) - - ##sleep 1 - end - photo_map = photo_map.merge!("photo_sizes" => photo_sizes_map) - - photo = photo - 1 - - #barf(photo_map.to_json, 3) - #barf(JSON.pretty_generate(photo_map), 3) - File.open(photo_cache_dir + "/meta.json", 'wb') {|f| f.write(JSON.pretty_generate(photo_map))} - ##exit - ##sleep 1 - end - - page = page + 1 - ##sleep 1 -end - -##puts(overall_tags.sort) -##puts(JSON.pretty_generate(overall_tags.sort)) - -barf("Finished...!", 2) diff --git a/tmp/flickr.rb b/tmp/flickr.rb deleted file mode 100644 index 7923d96..0000000 --- a/tmp/flickr.rb +++ /dev/null @@ -1,279 +0,0 @@ -# -# https://www.flickr.com/services/api/ -# - -require 'adler32' -require 'digest' -require 'digest/crc32' -#require 'easy-crc' -require 'flickraw' -require 'openssl' - -FlickRaw.api_key = "" -FlickRaw.shared_secret = "" - -flickr.access_token = "" -flickr.access_secret = "" - -def titleize(str) - str.split(/ |\_/).map(&:capitalize).join(" ") -end - -def get_count(f, pp, pl) - # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html - count = f.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => pp, :page => pl).length -end - -login = flickr.test.login -puts "You are now authenticated as #{login.username}" - -pp = 20 -pl = 0 -photo_count = 0 -page_count = 0 -print "Counting Pages...\n" -while get_count(flickr, pp, pl) != 0 do - pl = pl + 1 - c = get_count(flickr, pp, pl) - photo_count = photo_count + c - print "." - STDOUT.flush -end -page_count = pl - 1 -print "\n" -print "Counted: " + page_count.to_s() + " pages..." -print "\n" -print "Counted: " + photo_count.to_s() + " photos..." -print "\n" - -tmp_page1 = 1 -paging_code1 = "

Pages

" -while tmp_page1 <= page_count do - if tmp_page1 < 2 - paging_code1 = paging_code1 + "1" - else - paging_code1 = paging_code1 + "" + tmp_page1.to_s + "" - end - tmp_page1 = tmp_page1 + 1 - #print "." - #STDOUT.flush -end -paging_code1 = paging_code1 + "
" - - -puts "Creating Pages...\n" -page = 1 - -while page <= page_count do - #print "Creating Pagination... for page" + page.to_s + "\n" - print "Page: " + page.to_s + "\n" - tmp_page = 1 - paging_code = "

Pages

" - while tmp_page <= page_count do - if tmp_page == page - paging_code = paging_code + "" + tmp_page.to_s + "" - elsif tmp_page != page - paging_code = paging_code + "" + tmp_page.to_s + "" - end - tmp_page = tmp_page + 1 - #print "." - STDOUT.flush - end - paging_code = paging_code + "
" - #print "\n" - - #print "." - #STDOUT.flush - - # https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html - list = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => pp, :page => page) - - h_cmd = "" - if page < 2 - h_cmd = "mkdir -p photos/" + page.to_s + "; " - h_cmd = h_cmd + "echo '---\nlayout: photos\nstatus: publish\npublished: true\nnoToc: true\ntitle: Photos\n---\n' > photos/index.html; " - h_cmd = h_cmd + "echo '---\nlayout: photos\nstatus: publish\npublished: true\nnoToc: true\ntitle: Photos\n---\n' > photos/1/index.html; " - h_cmd = h_cmd + "echo '
' >> photos/" + page.to_s + "/index.html; " - p_cmd = p_cmd + "echo '" + paging_code + "' >> photos/" + page.to_s + "/index.html; " - end - wasGood = system( p_cmd ) - sleep 1 - p_cmd = "" - - page = page + 1 -end - -print "\n" -print "Finished...!\n" -