hanez/fgg
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Moved rgg.rb to tmp/ since the new Python edition is more mature now.

Browse source
This commit is contained in:
Johannes Findeisen 2022-11-21 01:13:40 +01:00
commit 0d837605ed
1 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

317
tmp/fgg.rb Normal file
View file

@ -0,0 +1,317 @@
#!/bin/env ruby
# Copyright (c) 2022 Johannes Findeisen
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is furnished
# to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# fgg - A gallery generator for static site generators using Flickr as data source.
#
# THIS IS THE PROTOTYPE FOR THE PYTHON VERSION OF THIS SCRIPT! IT EXISTS BECAUSE I AM
# USING JEKYLL SINCE SOME YEARS FOR SITE GENERATION. SINCE I AM SWITCHING TO NIKOLA
# WHICH IS WRITTEN IN PYTHON THIS WILL BECOME OBSOLETE ASAP SINCE MY FAVORITE LANGUAGE
# IS PYTHON AND RUBY REALLY SUCKS IN MANY WAYS...
#
# This is a port from a prototype I have running since some years. I want to port
# this to Python in the future.
#
# Flickr API documentation
# https://www.flickr.com/services/api/
require 'date'
require 'digest'
require 'digest/crc32'
require 'fileutils'
require 'flickraw'
require 'json'
require 'openssl'
FlickRaw.api_key = ENV["FLICKR_API_KEY"]
FlickRaw.shared_secret = ENV["FLICKR_SHARED_SECRET"]
flickr.access_token = ENV["FLICKR_ACCESS_TOKEN"]
flickr.access_secret = ENV["FLICKR_ACCESS_SECRET"]
# error = 0
# warning = 1
# info = 2
# debug = 3
def barf(message, log)
if log == 0
puts("ERROR: " + message)
end
if log == 1
puts("WARNING: " + message)
end
if log == 2
puts("INFO: " + message)
end
if log == 3
puts("DEBUG: " + message)
end
end
def fill_zeros(number, digit_count=3)
format_string = "%0#{digit_count}d"
return format_string % (number % 10**digit_count)
end
def get_count(flickr, photos_per_page, page)
# https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html
count = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page).length
end
barf("Logging in to Flickr...", 2)
login = flickr.test.login
barf("You are now authenticated as #{login.username}", 2)
# the timezone difference to UTC
tz_diff = "+01:00"
page = 1
page_count = 0
photo_count = 0
photos_per_page = 500
barf("Counting Pages...", 2)
while get_count(flickr, photos_per_page, page) != 0 do
count = get_count(flickr, photos_per_page, page)
photo_count = photo_count + count
print "."
STDOUT.flush
page = page + 1
page_count = page_count + 1
end
barf("", 2)
barf("Page Count: " + page_count.to_s(), 2)
barf("Photo Count: " + photo_count.to_s(), 2)
overall_tags = []
page = 1
photo = photo_count
photo_map = {}
while page <= page_count do
barf("Page: " + page.to_s, 2)
# https://www.flickr.com/services/api/flickr.photos.getWithoutGeoData.html
photo_list = flickr.photos.getWithoutGeoData(:sort => 'date-taken-desc', :per_page => photos_per_page, :page => page)
photo_list.each_with_index do
|flickr_photo|
barf("Photo: " + photo.to_s, 2)
##puts(flickr_photo.inspect)
##exit
if File.file?("cache/flickraw/" + flickr_photo.id + "/meta")
flickr_photo_info = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/meta"))
else
# https://www.flickr.com/services/api/flickr.photos.getInfo.html
flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id)
FileUtils.mkdir_p("cache/flickraw/" + flickr_photo.id)
File.open("cache/flickraw/" + flickr_photo.id + "/meta", 'wb') {|f| f.write(Marshal.dump(flickr_photo_info))}
end
#flickr_photo_info = flickr.photos.getInfo(:photo_id => flickr_photo.id)
#puts(flickr_photo_info.inspect)
#exit
photo_timestamp = DateTime.parse(flickr_photo_info.dates.taken.to_s + tz_diff).to_time.to_i
photo_suffix = fill_zeros(photo, 7)
photo_cache_dir = "cache/photos/" + photo_timestamp.to_s + "-" + photo_suffix
if Dir.exists?(photo_cache_dir)
barf("File exists: " + photo_cache_dir, 1)
else
FileUtils.mkdir_p(photo_cache_dir)
end
barf("Processing photo number: " + photo.to_s, 2)
barf("Flickr photo id: " + flickr_photo.id, 2)
photo_map = photo_map.merge!("photo_flickr_id" => flickr_photo.id)
barf("Flickr photo owner: " + flickr_photo.owner, 2)
photo_map = photo_map.merge!("photo_flickr_owner" => flickr_photo.owner)
barf("Flickr photo title: " + flickr_photo.title, 2)
photo_map = photo_map.merge!("photo_flickr_title" => flickr_photo.title)
##barf("Flickr photo description: " + flickr_photo_info.description.partition("[").first, 2)
##photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description.partition("[").first)
barf("Flickr photo description: " + flickr_photo_info.description, 2)
photo_map = photo_map.merge!("photo_flickr_description" => flickr_photo_info.description)
barf("Flickr photo location: " + flickr_photo_info.owner["location"], 2)
photo_map = photo_map.merge!("photo_flickr_location" => flickr_photo_info.owner["location"])
barf("Flickr photo username: " + flickr_photo_info.owner["username"], 2)
photo_map = photo_map.merge!("photo_flickr_username" => flickr_photo_info.owner["username"])
barf("Date taken: " + flickr_photo_info.dates.taken.to_s, 2)
photo_map = photo_map.merge!("photo_flickr_date_taken" => "" + flickr_photo_info.dates.taken.to_s + "")
barf("Date taken timestamp: " + photo_timestamp.to_s, 2)
photo_map = photo_map.merge!("photo_flickr_date_taken_timestamp" => photo_timestamp)
barf("Date uploaded: " + flickr_photo_info.dateuploaded.to_s, 2)
photo_map = photo_map.merge!("photo_flickr_date_uploaded_timestamp" => flickr_photo_info.dateuploaded.to_i)
##puts flickr_photo_info.dates.inspect
##puts flickr_photo_info.inspect
# START ########################################################################################################
# NEVER CHANGE THE FOLLOWING LINES (UNTIL "# END") EXPECT YOU REALLY NEED OR WANT TO CHANGE THE PHOTO ID!!!
# ALL LINKS TO THIS PHOTO WILL BECOME BROKEN!!! - hanez
# https://docs.ruby-lang.org/en/master/OpenSSL/Digest.html
digest1 = OpenSSL::Digest.new("sha256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
digest2 = OpenSSL::Digest.new("sha512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
digest3 = OpenSSL::Digest.new("sha3-256", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
digest4 = OpenSSL::Digest.new("sha3-512", flickr_photo.id.to_s + flickr_photo_info.dates.taken.to_s)
photo_id = Digest::CRC32c.hexdigest(digest1.to_s + digest2.to_s + digest3.to_s + digest4.to_s)
barf("Photo id: " + photo_id, 2)
photo_map = photo_map.merge!("photo_id" => photo_id)
# END ##########################################################################################################
FileUtils.mkdir_p "photo/" + photo_id
## https://westonganger.com/posts/get-substring-between-two-strings-or-characters-in-ruby
## get string between last "[" and last "]":
##tags = flickr_photo_info.description[/.*\[(.*)\]/, 1]
##if tags
## barf("Flickr photo tags: " + tags, 2)
## photo_map = photo_map.merge! "photo_flickr_tags" => tags
##else
## barf("Flickr photo tags: " + "", 2)
## photo_map = photo_map.merge! "photo_flickr_tags" => ""
##end
##puts(flickr_photo_info.tags.inspect)
if File.file?(photo_cache_dir + "/tags.array")
tags = Marshal.load(File.binread(photo_cache_dir + "/tags.array"))
else
tags = []
flickr_photo_info.tags.each_with_index do
|flickr_photo_tags|
tags.push(flickr_photo_tags.raw.downcase)
end
File.open(photo_cache_dir + "/tags.array", 'wb') {|f| f.write(Marshal.dump(tags))}
end
overall_tags = overall_tags | tags
photo_map = photo_map.merge!("photo_flickr_tags" => tags)
# convert 2004-11-19 12:51:19 to 20041119125119
photo_date = flickr_photo_info.dates.taken
photo_date = photo_date.to_s()
photo_date["-"] = ""
photo_date["-"] = ""
photo_date[" "] = ""
photo_date[":"] = ""
photo_date[":"] = ""
barf("Photo date: " + photo_date, 2)
photo_map = photo_map.merge!("photo_date" => photo_date)
photo_map = photo_map.merge!("photo_timezone" => "UTC" + tz_diff)
# the filename should be optimized to create a more beautiful filename
filename = String.new(flickr_photo_info.title.tr("\s", "-"))
filename = filename.delete("\'\"\.\s\,\!\/\:\(\)\[\]\;\?\&\<\>\$\%")
filename = filename.gsub("--", "-")
barf("Filename: " + filename, 2)
photo_map = photo_map.merge!("photo_filename" => filename)
if File.file?("cache/flickraw/" + flickr_photo.id + "/sizes")
sizes = Marshal.load(File.binread("cache/flickraw/" + flickr_photo.id + "/sizes"))
else
# https://www.flickr.com/services/api/flickr.photos.getSizes.html
sizes = flickr.photos.getSizes(:photo_id => flickr_photo.id)
File.open("cache/flickraw/" + flickr_photo.id + "/sizes", 'wb') {|f| f.write(Marshal.dump(sizes))}
end
#puts(sizes.inspect)
#exit
photo_sizes_map = {}
sizes.each_with_index do
|flickr_photo_sizes|
photo_size_map = {}
size = flickr_photo_sizes.url[-2, 1]
if flickr_photo_sizes.label == "Square"
size = "x"
end
barf("Photo size: " + size, 3)
photo_size_map = photo_size_map.merge!("size" => size)
photo_target_size = flickr_photo_sizes.label.tr(" ", "-")
photo_target_file = "photo/" + photo_id + "/" + filename + "-" + photo_date + "-" + photo_suffix + "-" + photo_target_size + ".jpg"
if !File.file?(photo_target_file)
cmd = "wget -U Foo -O " + photo_target_file + " " + flickr_photo_sizes.source
exec = `#{cmd}`
end
barf("Photo target: " + photo_target_file, 3)
photo_size_map = photo_size_map.merge!("target_file" => photo_target_file)
photo_digest = Digest::SHA1.file(photo_target_file)
photo_digest = photo_digest.hexdigest
photo_digest = photo_digest.to_s()
barf("Photo digest (SHA1): " + photo_digest, 3)
photo_size_map = photo_size_map.merge!("digest_sha1" => photo_digest)
photo_digest = Digest::SHA256.file(photo_target_file)
photo_digest = photo_digest.hexdigest
photo_digest = photo_digest.to_s()
barf("Photo digest (SHA256): " + photo_digest, 3)
photo_size_map = photo_size_map.merge!("digest_sha256" => photo_digest)
photo_digest = Digest::SHA512.file(photo_target_file)
photo_digest = photo_digest.hexdigest
photo_digest = photo_digest.to_s()
barf("Photo digest (SHA512): " + photo_digest, 3)
photo_size_map = photo_size_map.merge!("digest_sha512" => photo_digest)
barf("Photo size label: " + flickr_photo_sizes.label, 3)
photo_size_map = photo_size_map.merge!("label" => flickr_photo_sizes.label)
barf("Photo size width: " + flickr_photo_sizes.width.to_s, 3)
photo_size_map = photo_size_map.merge!("width" => flickr_photo_sizes.width)
barf("Photo size height: " + flickr_photo_sizes.height.to_s, 3)
photo_size_map = photo_size_map.merge!("height" => flickr_photo_sizes.height)
photo_sizes_map = photo_sizes_map.merge!(size => photo_size_map)
##sleep 1
end
photo_map = photo_map.merge!("photo_sizes" => photo_sizes_map)
photo = photo - 1
#barf(photo_map.to_json, 3)
#barf(JSON.pretty_generate(photo_map), 3)
File.open(photo_cache_dir + "/meta.json", 'wb') {|f| f.write(JSON.pretty_generate(photo_map))}
##exit
##sleep 1
end
page = page + 1
##sleep 1
end
##puts(overall_tags.sort)
##puts(JSON.pretty_generate(overall_tags.sort))
barf("Finished...!", 2)