Added downloading of photos and SHA checksum generation and a lot of optimizations.

README update.

Raised version to 0.1.0 because of new features.

README.md

@@ -6,6 +6,12 @@ fgg is a free gallery generator for static site generators like Hugo, Jekyll, Ni
 
 More to come...
 
+## Requirenents:
+
+- Python 3.* (I don't now the concrete version right now. I use only Python 3.10 at the moment. Could be that fgg is 
+  running with an older version, but it could also be that required libs do not. Please report your Python version to 
+  me if fgg works for you to help me to create a compatibility list.)
+
 ## To do:
 
 - Error handling. Not done in any way so far... :|

fgg

@@ -35,9 +35,10 @@ import hashlib
 import json
 import pathlib
 import os
+import requests
 import time
 
-__version__ = "0.0.3"
+__version__ = "0.1.0"
 
 flickr_api_key = os.environ['FLICKR_API_KEY']
 flickr_api_secret = os.environ['FLICKR_API_SECRET']
@@ -51,9 +52,30 @@ auth_handler = flickr.auth.AuthHandler(
 flickr.set_auth_handler(auth_handler)
 user = flickr.test.login()
 
-debug_max_photos = 10
+# Some settings for execution. These should be configurable with CLI arguments.
+# Set max number of photos to process. 0 = No limit.
+debug_max_photos = 0
+# The timezone
 timezone = 'UTC+01:00'
 
+
+def hash_file(filename, algorithm='sha1'):
+    if algorithm == 'sha1':
+        h = hashlib.sha1()
+    if algorithm == 'sha256':
+        h = hashlib.sha256()
+    if algorithm == 'sha512':
+        h = hashlib.sha512()
+
+    with open(filename, 'rb') as file:
+        chunk = 0
+        while chunk != b'':
+            chunk = file.read(1024)
+            h.update(chunk)
+
+    return h.hexdigest()
+
+
 print('Creating gallery...')
 print('Counting pages...')
 
@@ -89,13 +111,24 @@ while photo_page <= page_count:
     for flickr_photo in flickr_photo_list:
         print(str(i) + "/" + str(photo_count))
         photo = {}
-        if os.path.exists('db/' + flickr_photo.id + '/meta.json'):
+        if os.path.exists('cache/' + flickr_photo.id + '/meta.json'):
-            with open('db/' + flickr_photo.id + '/meta.json', 'rb') as photo_json_file:
+            with open('cache/' + flickr_photo.id + '/meta.json', 'rb') as f:
-                ##print('Loading from cache: cache/flickr/' + flickr_photo.id + '/meta.json')
+                print('Loading photo from cache file: cache/' + flickr_photo.id + '/meta.json')
-                photo = json.load(photo_json_file)
+                photo = json.load(f)
+                f.close()
         else:
+            print('Creating photo cache file: cache/' + flickr_photo.id + '/meta.json')
             flickr_photo_info = flickr.Photo.getInfo(flickr_photo)
 
+            photo['date_posted_timestamp'] = int(flickr_photo_info['posted'])
+            photo['date_taken'] = flickr_photo_info['taken']
+            photo['date_taken_timestamp'] = int(time.mktime(
+                time.strptime(flickr_photo_info['taken'], '%Y-%m-%d %H:%M:%S')))
+
+            photo['date_updated_timestamp'] = int(flickr_photo_info['lastupdate'])
+            photo['date_uploaded_timestamp'] = int(flickr_photo_info['dateuploaded'])
+            photo['description'] = flickr_photo_info['description']
+
             ########################################################################################
             # START ################################################################################
             # NEVER CHANGE THE FOLLOWING LINES (UNTIL "# END") EXPECT YOU REALLY NEED OR WANT TO
@@ -118,51 +151,46 @@ while photo_page <= page_count:
             # END ##################################################################################
             ########################################################################################
 
-            pathlib.Path('db/' + flickr_photo.id).mkdir(parents=True, exist_ok=True)
-            with open('db/' + flickr_photo.id + '/meta.json', 'w') as photo_json_file:
-                photo['title'] = flickr_photo.title
-                photo['date_updated_timestamp'] = int(flickr_photo_info['lastupdate'])
-                photo['date_posted_timestamp'] = int(flickr_photo_info['posted'])
-                photo['date_taken'] = flickr_photo_info['taken']
-                photo['date_taken_timestamp'] = int(time.mktime(
-                    time.strptime(flickr_photo_info['taken'], '%Y-%m-%d %H:%M:%S')))
-
-                photo['date_uploaded_timestamp'] = int(flickr_photo_info['dateuploaded'])
-                photo['description'] = flickr_photo_info['description']
-
             photo['fgg_id'] = str(photo['date_taken_timestamp']) + '-' + str(i).rjust(7, '0')
             photo['flickr_id'] = flickr_photo.id
-
             photo['photo_number'] = i
 
             flickr_photo_tags = []
             for flickr_photo_tag in flickr_photo_info['tags']:
                 flickr_photo_tags.append(flickr_photo_tag['text'])
 
+            photo['tags'] = []
             photo['tags'] = flickr_photo_tags
-                photo['tags'].sort()
+            #photo['tags'].sort()
             # TODO: This does not work and need be fixed...!
-                overall_tags = list(set(overall_tags + photo['tags']))
+            overall_tags = list(set(overall_tags + flickr_photo_tags))
 
+            photo['target_path'] = 'photo/' + photo['digest']
             photo['timezone'] = timezone
+            photo['title'] = flickr_photo.title
 
+            if os.path.exists('cache/' + flickr_photo.id + '/sizes.json'):
+                with open('cache/' + flickr_photo.id + '/sizes.json', 'rb') as f:
+                    print('Loading sizes from cache file: cache/' + flickr_photo.id + '/sizes.json')
+                    flickr_photo_sizes = json.load(f)
+                    f.close()
+            else:
+                pathlib.Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True)
+                with open('cache/' + flickr_photo.id + '/sizes.json', 'w') as f:
+                    print('Creating sizes cache file: cache/' + flickr_photo.id + '/sizes.json')
                     flickr_photo_sizes = flickr_photo.getSizes()
+                    json.dump(flickr_photo_sizes, f, ensure_ascii=False, indent=2, sort_keys=True)
+                    f.close()
+
             photo['sizes'] = {}
             for flickr_photo_size in flickr_photo_sizes:
                 flickr_size = flickr_photo_sizes[flickr_photo_size]['url'].split("/")[-2] \
                     .replace('/', '')
-
                 if flickr_photo_sizes[flickr_photo_size]['label'] == "Square":
                     flickr_size = "x"
 
                 photo['sizes'][flickr_size] = {}
-
-                    # Remove all unneeded keys
-                    flickr_photo_sizes[flickr_photo_size].pop('media')
-                    flickr_photo_sizes[flickr_photo_size].pop('source')
-                    flickr_photo_sizes[flickr_photo_size].pop('url')
                 photo['sizes'][flickr_size] = flickr_photo_sizes[flickr_photo_size]
-
                 photo['sizes'][flickr_size]['size'] = flickr_size
 
                 # Create target photo folder dir and filenames
@@ -193,32 +221,63 @@ while photo_page <= page_count:
                 target_file_name = target_file_name.rstrip()
                 target_file_name = target_file_name.replace(' ', '-')
 
-                    target_file_name = target_file_name + '-' + photo['fgg_id'] + \
+                target_file_name = target_file_name + '-' + photo['fgg_id'] + '-' + \
-                        '-' + flickr_photo_sizes[flickr_photo_size]['label'].replace(' ', '-')
+                    flickr_photo_sizes[flickr_photo_size]['label'].replace(' ', '-')
+
                 target_file_name = target_file_name + '.jpg'
 
                 photo['sizes'][flickr_size]['target_file_name'] = target_file_name
-                    photo['sizes'][flickr_size]['target_file'] = 'photo/' + \
+                photo['sizes'][flickr_size]['target_file'] = \
-                        photo['digest'] + '/' + \
+                    'photo/' + photo['digest'] + '/' + \
                     photo['sizes'][flickr_size]['target_file_name']
 
-                    # TODO: Download photo file if not exists.
+                # Download photo file if it not exists.
+                if not os.path.exists(photo['sizes'][flickr_size]['target_file']):
+                    url = flickr_photo_sizes[flickr_photo_size]['source']
+                    req = requests.get(url)
+                    print('Downloading: ' + url)
+                    pathlib.Path(photo['sizes'][flickr_size]['target_file'].rsplit('/', 1)[0]) \
+                        .mkdir(parents=True, exist_ok=True)
 
-                    # TODO: Create SHA sums for each photo file and add them to.
+                    print('Writing file to: ' + photo['sizes'][flickr_size]['target_file'])
-                    #  photo['sizes'][flickr_size][...]
+                    with open(photo['sizes'][flickr_size]['target_file'], 'wb') as f:
-                    photo['sizes'][flickr_size]['sha1'] = 'SHA1 of file: ' + \
+                        f.write(req.content)
-                        photo['sizes'][flickr_size]['target_file']
+                        f.close()
+                        print("Download Completed!")
 
-                    photo['sizes'][flickr_size]['sha256'] = 'SHA256 of file: ' + \
+                # Create checksums for each photo file and add them to
-                        photo['sizes'][flickr_size]['target_file']
+                # photo['sizes'][flickr_size][...].
+                photo['sizes'][flickr_size]['sha1'] = \
+                    hash_file(photo['sizes'][flickr_size]['target_file'], 'sha1')
 
-                    photo['sizes'][flickr_size]['sha512'] = 'SHA512 of file: ' + \
+                photo['sizes'][flickr_size]['sha256'] = \
-                        photo['sizes'][flickr_size]['target_file']
+                    hash_file(photo['sizes'][flickr_size]['target_file'], 'sha256')
 
-                json.dump(photo, photo_json_file, ensure_ascii=False, indent=2, sort_keys=True)
+                photo['sizes'][flickr_size]['sha512'] = \
+                    hash_file(photo['sizes'][flickr_size]['target_file'], 'sha512')
+
+                # Remove all unneeded keys
+                photo['sizes'][flickr_size].pop('media')
+                photo['sizes'][flickr_size].pop('source')
+                photo['sizes'][flickr_size].pop('url')
+
+            pathlib.Path('cache/' + flickr_photo.id).mkdir(parents=True, exist_ok=True)
+            with open('cache/' + flickr_photo.id + '/meta.json', 'w') as f:
+                json.dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True)
+                f.close()
+
+            with open(photo['target_path'] + '/meta.json', 'w') as f:
+                json.dump(photo, f, ensure_ascii=False, indent=2, sort_keys=True)
+                f.close()
 
         i -= 1
         if i == photo_count - debug_max_photos:
             exit(0)
 
     photo_page += 1
+
+photos = {}
+photos['tags'] = overall_tags
+with open('photos/meta.json', 'w') as f:
+    json.dump(photos, f, ensure_ascii=False, indent=2, sort_keys=True)
+    f.close()

requirements.txt

@@ -1,2 +1,3 @@
-crc32c==2.3
+crc32c~=2.3
-flickr-api==0.7.5
+flickr-api~=0.7.5
+requests~=2.28.1