hanez/fgg
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity
fgg/fgg

304 lines
14 KiB
Python
Executable file
Raw Blame History

#!/bin/env python
# fgg - A free gallery generator for static site generators like Hugo, Jekyll,
# Nikola etc. using Flickr as data source.
# Source repository: https://git.unixpeople.org/hanez/fgg
# Copyright (c) 2022 Johannes Findeisen
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is furnished
# to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# Flickr API documentation
# https://www.flickr.com/services/api/
# https://github.com/alexis-mignon/python-flickr-api
# https://github.com/alexis-mignon/python-flickr-api/wiki/API-reference
# This will become the next generation of fgg!
import flickr_api as flickr
from crc32c import crc32c
from hashlib import sha1, sha256, sha512, sha3_256, sha3_512
from json import dump, load
from os import environ
from os.path import exists
from pathlib import Path
from requests import get
from time import mktime, strptime
__version__ = "0.1.3"
flickr_api_key = environ['FLICKR_API_KEY']
flickr_api_secret = environ['FLICKR_API_SECRET']
flickr.set_keys(api_key=flickr_api_key,
api_secret=flickr_api_secret)
auth_handler = flickr.auth.AuthHandler(
access_token_key=environ['FLICKR_ACCESS_TOKEN'],
access_token_secret=environ['FLICKR_ACCESS_SECRET'])
flickr.set_auth_handler(auth_handler)
user = flickr.test.login()
# Some settings for execution. These should be configurable with CLI arguments.
# Set max number of photos to process. 0 = No limit.
debug_max_photos = 0
# The timezone
timezone = 'UTC+01:00'
def hash_file(filename, algorithm='sha1'):
if algorithm == 'sha1':
h = sha1()
if algorithm == 'sha256':
h = sha256()
if algorithm == 'sha512':
h = sha512()
with open(filename, 'rb') as file:
chunk = 0
while chunk != b'':
chunk = file.read(1024)
h.update(chunk)
return h.hexdigest()
print('Creating gallery...')
print('Counting pages...')
page_count = 0
photo_count = 0
photo_page = 1
photos_per_page = 500
while True:
count = len(flickr.Photo.getWithoutGeoData(sort='date-taken-desc',
per_page=photos_per_page,
page=photo_page))
if not count:
break
photo_count = photo_count + count
print('.', flush=True, end='')
photo_page += 1
page_count += 1
print()
print('Page count: ' + str(page_count))
print('Photo count: ' + str(photo_count))
photo_number = photo_count
photo_process_number = 1
overall_tags = []
photo_page = 1
while photo_page <= page_count:
print("Page: " + str(photo_page))
# TODO: Make this cacheable. Use JSON for that and merge results of all pages to one dict.
# E.g.: flickr_photo_list should be a JSON object not a flickr_api object.
# Then there is no need to loop pages around all code below but only for generating the dict.
# It will make fgg to use only the cache without any Flickr API calls except the count pages
# code above. But I will find to a full cache based solution some day... ;)
flickr_photo_list = user.getPhotos(sort='date-taken-desc',
per_page=photos_per_page,
page=photo_page)
for flickr_photo in flickr_photo_list:
print(str(photo_process_number) + "/" + str(photo_count))
photo = {}
if exists('cache/' + flickr_photo.id + '/meta.json'):
with open('cache/' + flickr_photo.id + '/meta.json', 'rb') as f:
print('Loading photo from cache file: cache/' + flickr_photo.id + '/meta.json')
photo = load(f)
f.close()
else:
print('Creating photo cache file: cache/' + flickr_photo.id + '/meta.json')
flickr_photo_info = flickr.Photo.getInfo(flickr_photo)
photo['date_posted_timestamp'] = int(flickr_photo_info['posted'])
photo['date_taken'] = flickr_photo_info['taken']
photo['date_taken_timestamp'] = int(mktime(
strptime(flickr_photo_info['taken'], '%Y-%m-%d %H:%M:%S')))
photo['date_updated_timestamp'] = int(flickr_photo_info['lastupdate'])
photo['date_uploaded_timestamp'] = int(flickr_photo_info['dateuploaded'])
photo['description'] = flickr_photo_info['description']
# START ################################################################################
# NEVER CHANGE THE FOLLOWING LINES (UP TO "# END") UNLESS YOU REALLY NEED OR WANT TO
# CHANGE THE PHOTO ID!!! ALL LINKS TO THE PHOTO PAGE WILL BECOME BROKEN!!! - hanez
digest1 = sha256(str.encode(str(flickr_photo.id) +
str(flickr_photo_info['taken']))).hexdigest()
digest2 = sha512(str.encode(str(flickr_photo.id) +
str(flickr_photo_info['taken']))).hexdigest()
digest3 = sha3_256(str.encode(str(flickr_photo.id) +
str(flickr_photo_info['taken']))).hexdigest()
digest4 = sha3_512(str.encode(str(flickr_photo.id) +
str(flickr_photo_info['taken']))).hexdigest()
digest = str(str(digest1) + str(digest2) + str(digest3) + str(digest4))
digest = str(hex(crc32c.crc32c(bytes(digest, 'ascii')))).replace('0x', '')
photo['digest'] = digest
# END ##################################################################################
photo['fgg_id'] = str(photo['date_taken_timestamp']) + '-' + \
str(photo_number).rjust(7, '0')
photo['flickr_id'] = flickr_photo.id
photo['photo_number'] = photo_number
flickr_photo_tags = []
for flickr_photo_tag in flickr_photo_info['tags']:
flickr_photo_tags.append(flickr_photo_tag['text'])
photo['tags'] = []
photo['tags'] = flickr_photo_tags
#photo['tags'].sort()
# TODO: This does not work and need be fixed...!
overall_tags = list(set(overall_tags + flickr_photo_tags))
photo['target_path'] = 'photo/' + photo['digest']
photo['timezone'] = timezone
photo['title'] = flickr_photo.title
if exists('cache/' + flickr_photo.id + '/sizes.json'):
with open('cache/' + flickr_photo.id + '/sizes.json', 'rb') as f:
print('Loading sizes from cache file: cache/' + flickr_photo.id + '/sizes.json')
flickr_photo_sizes = load(f)
f.close()
else:
Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True)
with open('cache/' + flickr_photo.id + '/sizes.json', 'w') as f:
print('Creating sizes cache file: cache/' + flickr_photo.id + '/sizes.json')
flickr_photo_sizes = flickr_photo.getSizes()
dump(flickr_photo_sizes, f, ensure_ascii=False, indent=2, sort_keys=True)
f.close()
photo['sizes'] = {}
for flickr_photo_size in flickr_photo_sizes:
flickr_size = flickr_photo_sizes[flickr_photo_size]['url'].split("/")[-2] \
.replace('/', '')
if flickr_photo_sizes[flickr_photo_size]['label'] == "Square":
flickr_size = "x"
photo['sizes'][flickr_size] = {}
photo['sizes'][flickr_size] = flickr_photo_sizes[flickr_photo_size]
photo['sizes'][flickr_size]['size'] = flickr_size
# Create target photo folder dir and filenames
# TODO: The string replacement should be optimized to make better file names.
target_file_name = flickr_photo.title.replace(':)', '')
target_file_name = target_file_name.replace(':D', '')
target_file_name = target_file_name.replace(';)', '')
target_file_name = target_file_name.replace(':-)', '')
target_file_name = target_file_name.replace(':-D', '')
target_file_name = target_file_name.replace(';-)', '')
target_file_name = target_file_name.replace('\'', '')
target_file_name = target_file_name.replace('\"', '')
# Maybe not remove all dots too...?
target_file_name = target_file_name.replace('.', '')
target_file_name = target_file_name.replace(',', '')
target_file_name = target_file_name.replace('!', '')
target_file_name = target_file_name.replace(' / ', '-')
target_file_name = target_file_name.replace('/ ', '-')
target_file_name = target_file_name.replace('/', '-')
target_file_name = target_file_name.replace(':', '')
target_file_name = target_file_name.replace('(', '')
target_file_name = target_file_name.replace(')', '')
target_file_name = target_file_name.replace('[', '')
target_file_name = target_file_name.replace(']', '')
target_file_name = target_file_name.replace('{', '')
target_file_name = target_file_name.replace('}', '')
target_file_name = target_file_name.replace(';', '')
target_file_name = target_file_name.replace('?', '')
target_file_name = target_file_name.replace('&', '')
target_file_name = target_file_name.replace('<', '')
target_file_name = target_file_name.replace('>', '')
target_file_name = target_file_name.replace('$', '')
target_file_name = target_file_name.replace('%', '')
# Remove trailing whitespaces.
target_file_name = target_file_name.rstrip()
target_file_name = target_file_name.replace(' ', '-')
target_file_name = target_file_name.replace('-----', '-')
target_file_name = target_file_name.replace('----', '-')
target_file_name = target_file_name.replace('---', '-')
target_file_name = target_file_name.replace('--', '-')
target_file_name = target_file_name + '-' + photo['fgg_id'] + '-' + \
flickr_photo_sizes[flickr_photo_size]['label'].replace(' ', '-')
target_file_name = target_file_name + '.jpg'
photo['sizes'][flickr_size]['target_file_name'] = target_file_name
photo['sizes'][flickr_size]['target_file'] = \
'photo/' + photo['digest'] + '/' + \
photo['sizes'][flickr_size]['target_file_name']
# Download photo file if it not exists.
if not exists(photo['sizes'][flickr_size]['target_file']):
url = flickr_photo_sizes[flickr_photo_size]['source']
request = get(url)
print('Downloading file: ' + url)
Path(photo['sizes'][flickr_size]['target_file'].rsplit('/', 1)[0]) \
.mkdir(parents=True, exist_ok=True)
print('Writing file to: ' + photo['sizes'][flickr_size]['target_file'])
with open(photo['sizes'][flickr_size]['target_file'], 'wb') as f:
f.write(request.content)
f.close()
print("Download Completed!")
# Create checksums for each photo file and add them to
# photo['sizes'][flickr_size][...].
photo['sizes'][flickr_size]['sha1'] = \
hash_file(photo['sizes'][flickr_size]['target_file'], 'sha1')
photo['sizes'][flickr_size]['sha256'] = \
hash_file(photo['sizes'][flickr_size]['target_file'], 'sha256')
photo['sizes'][flickr_size]['sha512'] = \
hash_file(photo['sizes'][flickr_size]['target_file'], 'sha512')
# Remove all unneeded keys
photo['sizes'][flickr_size].pop('media')
photo['sizes'][flickr_size].pop('source')
photo['sizes'][flickr_size].pop('url')
Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True)
with open('cache/' + flickr_photo.id + '/meta.json', 'w') as f:
dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True)
f.close()
with open(photo['target_path'] + '/meta.json', 'w') as f:
dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True)
f.close()
photo_number -= 1
photo_process_number += 1
if photo_number == photo_count - debug_max_photos:
exit(0)
photo_page += 1
photos = {}
photos['tags'] = overall_tags
with open('photos/meta.json', 'w') as f:
dump(photos, f, ensure_ascii=False, indent=2, sort_keys=True)
f.close()
Reference in a new issue View git blame Copy permalink