From 418337004a79191e5f0fff3768e3ebda190bcda2 Mon Sep 17 00:00:00 2001 From: Johannes Findeisen Date: Mon, 21 Nov 2022 22:54:32 +0100 Subject: [PATCH] Import optimizations (not sure if this is helpful). Some fixes and comments. --- fgg | 107 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/fgg b/fgg index 3f109ca..3b2e26f 100755 --- a/fgg +++ b/fgg @@ -30,31 +30,27 @@ # https://github.com/alexis-mignon/python-flickr-api/wiki/API-reference # This will become the next generation of fgg! -import crc32c import flickr_api as flickr -import hashlib -import json -import pathlib -import os -import requests -import sys -import time -# Set the title for Linux and macOS. -sys.stdout.write("\x1b]2;fgg\x07") -# Set the title for Windows. -os.system("fgg") +from crc32c import crc32c +from hashlib import sha1, sha256, sha512, sha3_256, sha3_512 +from json import dump, load +from os import environ +from os.path import exists +from pathlib import Path +from requests import get +from time import mktime, strptime -__version__ = "0.1.2" +__version__ = "0.1.3" -flickr_api_key = os.environ['FLICKR_API_KEY'] -flickr_api_secret = os.environ['FLICKR_API_SECRET'] +flickr_api_key = environ['FLICKR_API_KEY'] +flickr_api_secret = environ['FLICKR_API_SECRET'] flickr.set_keys(api_key=flickr_api_key, api_secret=flickr_api_secret) auth_handler = flickr.auth.AuthHandler( - access_token_key=os.environ['FLICKR_ACCESS_TOKEN'], - access_token_secret=os.environ['FLICKR_ACCESS_SECRET']) + access_token_key=environ['FLICKR_ACCESS_TOKEN'], + access_token_secret=environ['FLICKR_ACCESS_SECRET']) flickr.set_auth_handler(auth_handler) user = flickr.test.login() @@ -68,11 +64,11 @@ timezone = 'UTC+01:00' def hash_file(filename, algorithm='sha1'): if algorithm == 'sha1': - h = hashlib.sha1() + h = sha1() if algorithm == 'sha256': - h = hashlib.sha256() + h = sha256() if algorithm == 'sha512': - h = hashlib.sha512() + h = sha512() with open(filename, 'rb') as file: chunk = 0 @@ -105,23 +101,29 @@ print() print('Page count: ' + str(page_count)) print('Photo count: ' + str(photo_count)) -i = photo_count +photo_number = photo_count +photo_process_number = 1 overall_tags = [] photo_page = 1 while photo_page <= page_count: print("Page: " + str(photo_page)) + # TODO: Make this cacheable. Use JSON for that and merge results of all pages to one dict. + # E.g.: flickr_photo_list should be a JSON object not a flickr_api object. + # Then there is no need to loop pages around all code below but only for generating the dict. + # It will make fgg to use only the cache without any Flickr API calls except the count pages + # code above. But I will find to a full cache based solution some day... ;) flickr_photo_list = user.getPhotos(sort='date-taken-desc', per_page=photos_per_page, page=photo_page) for flickr_photo in flickr_photo_list: - print(str(i) + "/" + str(photo_count)) + print(str(photo_process_number) + "/" + str(photo_count)) photo = {} - if os.path.exists('cache/' + flickr_photo.id + '/meta.json'): + if exists('cache/' + flickr_photo.id + '/meta.json'): with open('cache/' + flickr_photo.id + '/meta.json', 'rb') as f: print('Loading photo from cache file: cache/' + flickr_photo.id + '/meta.json') - photo = json.load(f) + photo = load(f) f.close() else: print('Creating photo cache file: cache/' + flickr_photo.id + '/meta.json') @@ -129,8 +131,8 @@ while photo_page <= page_count: photo['date_posted_timestamp'] = int(flickr_photo_info['posted']) photo['date_taken'] = flickr_photo_info['taken'] - photo['date_taken_timestamp'] = int(time.mktime( - time.strptime(flickr_photo_info['taken'], '%Y-%m-%d %H:%M:%S'))) + photo['date_taken_timestamp'] = int(mktime( + strptime(flickr_photo_info['taken'], '%Y-%m-%d %H:%M:%S'))) photo['date_updated_timestamp'] = int(flickr_photo_info['lastupdate']) photo['date_uploaded_timestamp'] = int(flickr_photo_info['dateuploaded']) @@ -139,26 +141,28 @@ while photo_page <= page_count: # START ################################################################################ # NEVER CHANGE THE FOLLOWING LINES (UP TO "# END") UNLESS YOU REALLY NEED OR WANT TO # CHANGE THE PHOTO ID!!! ALL LINKS TO THE PHOTO PAGE WILL BECOME BROKEN!!! - hanez - digest1 = hashlib.sha256(str.encode(str(flickr_photo.id) + - str(flickr_photo_info['taken']))).hexdigest() + digest1 = sha256(str.encode(str(flickr_photo.id) + + str(flickr_photo_info['taken']))).hexdigest() - digest2 = hashlib.sha512(str.encode(str(flickr_photo.id) + - str(flickr_photo_info['taken']))).hexdigest() + digest2 = sha512(str.encode(str(flickr_photo.id) + + str(flickr_photo_info['taken']))).hexdigest() - digest3 = hashlib.sha3_256(str.encode(str(flickr_photo.id) + - str(flickr_photo_info['taken']))).hexdigest() + digest3 = sha3_256(str.encode(str(flickr_photo.id) + + str(flickr_photo_info['taken']))).hexdigest() - digest4 = hashlib.sha3_512(str.encode(str(flickr_photo.id) + - str(flickr_photo_info['taken']))).hexdigest() + digest4 = sha3_512(str.encode(str(flickr_photo.id) + + str(flickr_photo_info['taken']))).hexdigest() digest = str(str(digest1) + str(digest2) + str(digest3) + str(digest4)) digest = str(hex(crc32c.crc32c(bytes(digest, 'ascii')))).replace('0x', '') photo['digest'] = digest # END ################################################################################## - photo['fgg_id'] = str(photo['date_taken_timestamp']) + '-' + str(i).rjust(7, '0') + photo['fgg_id'] = str(photo['date_taken_timestamp']) + '-' + \ + str(photo_number).rjust(7, '0') + photo['flickr_id'] = flickr_photo.id - photo['photo_number'] = i + photo['photo_number'] = photo_number flickr_photo_tags = [] for flickr_photo_tag in flickr_photo_info['tags']: @@ -174,17 +178,17 @@ while photo_page <= page_count: photo['timezone'] = timezone photo['title'] = flickr_photo.title - if os.path.exists('cache/' + flickr_photo.id + '/sizes.json'): + if exists('cache/' + flickr_photo.id + '/sizes.json'): with open('cache/' + flickr_photo.id + '/sizes.json', 'rb') as f: print('Loading sizes from cache file: cache/' + flickr_photo.id + '/sizes.json') - flickr_photo_sizes = json.load(f) + flickr_photo_sizes = load(f) f.close() else: - pathlib.Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True) + Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True) with open('cache/' + flickr_photo.id + '/sizes.json', 'w') as f: print('Creating sizes cache file: cache/' + flickr_photo.id + '/sizes.json') flickr_photo_sizes = flickr_photo.getSizes() - json.dump(flickr_photo_sizes, f, ensure_ascii=False, indent=2, sort_keys=True) + dump(flickr_photo_sizes, f, ensure_ascii=False, indent=2, sort_keys=True) f.close() photo['sizes'] = {} @@ -248,16 +252,16 @@ while photo_page <= page_count: photo['sizes'][flickr_size]['target_file_name'] # Download photo file if it not exists. - if not os.path.exists(photo['sizes'][flickr_size]['target_file']): + if not exists(photo['sizes'][flickr_size]['target_file']): url = flickr_photo_sizes[flickr_photo_size]['source'] - req = requests.get(url) - print('Downloading: ' + url) - pathlib.Path(photo['sizes'][flickr_size]['target_file'].rsplit('/', 1)[0]) \ + request = get(url) + print('Downloading file: ' + url) + Path(photo['sizes'][flickr_size]['target_file'].rsplit('/', 1)[0]) \ .mkdir(parents=True, exist_ok=True) print('Writing file to: ' + photo['sizes'][flickr_size]['target_file']) with open(photo['sizes'][flickr_size]['target_file'], 'wb') as f: - f.write(req.content) + f.write(request.content) f.close() print("Download Completed!") @@ -277,17 +281,18 @@ while photo_page <= page_count: photo['sizes'][flickr_size].pop('source') photo['sizes'][flickr_size].pop('url') - pathlib.Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True) + Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True) with open('cache/' + flickr_photo.id + '/meta.json', 'w') as f: - json.dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True) + dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True) f.close() with open(photo['target_path'] + '/meta.json', 'w') as f: - json.dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True) + dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True) f.close() - i -= 1 - if i == photo_count - debug_max_photos: + photo_number -= 1 + photo_process_number += 1 + if photo_number == photo_count - debug_max_photos: exit(0) photo_page += 1 @@ -295,5 +300,5 @@ while photo_page <= page_count: photos = {} photos['tags'] = overall_tags with open('photos/meta.json', 'w') as f: - json.dump(photos, f, ensure_ascii=False, indent=2, sort_keys=True) + dump(photos, f, ensure_ascii=False, indent=2, sort_keys=True) f.close()