Skip to content

Commit

Permalink
Merge pull request #2 from northfacejmb/specify-base-url
Browse files Browse the repository at this point in the history 
add region support
  • Loading branch information
@tofran
tofran authored Sep 30, 2024
2 parents dfe612c + d23ea9f commit 74596ba
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 93 deletions.
55 changes: 27 additions & 28 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,52 +27,51 @@ Features:

## Usage

<!--
To update the code block with the usage below:
1. Resize your terminal to 100 columns. On most systems just run: `stty cols 100 rows 50`.
2. Run `storyblok-assets-cleanup --help` to get the usage output.
-->

```
usage: storyblok-assets-cleanup [-h] [--delete | --no-delete] [--backup | --no-backup]
[--cache | --no-cache]
usage: storyblok-assets-cleanup [-h] [--token TOKEN] --space-id SPACE_ID
[--region {eu,us,ca,au,cn}] [--delete | --no-delete]
[--backup | --no-backup] [--backup-directory BACKUP_DIRECTORY]
[--cache | --no-cache] [--cache-directory CACHE_DIRECTORY]
[--continue-download-on-failure | --no-continue-download-on-failure]
[--space-id SPACE_ID] [--token TOKEN]
[--blacklisted-folder-paths BLACKLISTED_FOLDER_PATHS]
[--blacklisted-words BLACKLISTED_WORDS]
[--cache-directory CACHE_DIRECTORY]
[--backup-directory BACKUP_DIRECTORY]
storyblok-assets-cleanup an utility to delete unused assets.
options:
-h, --help show this help message and exit
--token TOKEN Storyblok personal access token, alternatively use the env var
STORYBLOK_PERSONAL_ACCESS_TOKEN.
--space-id SPACE_ID Storyblok space ID, alternatively use the env var STORYBLOK_SPACE_ID.
--region {eu,us,ca,au,cn}
Storyblok region (default: EU)
--delete, --no-delete
If we should delete assets, default to false.
--backup, --no-backup
If we should backup assets (to ./assets_backup/<SPACE_ID>), defaults to
true.
If we should backup assets (to the directory specified in `--backup-
directory`), defaults to true.
--backup-directory BACKUP_DIRECTORY
Backup directory, defaults to ./assets_backup.
--cache, --no-cache If we should use cache the assets index. Defaults to True (recommended).
--cache-directory CACHE_DIRECTORY
Cache directory, defaults to ./cache.
--continue-download-on-failure, --no-continue-download-on-failure
If we should continue if the download of an asset fails. Defaults to true.
--space-id SPACE_ID Storyblok space ID, alternatively use the env var STORYBLOK_SPACE_ID.
--token TOKEN Storyblok personal access token, alternatively use the env var
STORYBLOK_PERSONAL_ACCESS_TOKEN.
--blacklisted-folder-paths BLACKLISTED_FOLDER_PATHS
Comma separated list of filepaths that should be ignored. Alternatively use
the env var BLACKLISTED_ASSET_FOLDER_PATHS. Default to none/empty list.
Comma separated list of filepaths that should be ignored. Alternatively
use the env var BLACKLISTED_ASSET_FOLDER_PATHS. Default to none/empty
list.
--blacklisted-words BLACKLISTED_WORDS
Comma separated list of words that should be used to ignore assets when they
are contained in its filename. Alternatively use the env var
Comma separated list of words that should be used to ignore assets when
they are contained in its filename. Alternatively use the env var
BLACKLISTED_ASSET_FILENAME_WORDS. Default to none/empty list.
--cache-directory CACHE_DIRECTORY
Cache directory, defaults to ./cache.
--backup-directory BACKUP_DIRECTORY
Backup directory, defaults to ./assets_backup.
```

## Development
Expand Down
159 changes: 94 additions & 65 deletions storyblok_assets_cleanup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,52 @@

import requests

_storyblok_space_id = None
_storyblok_personal_access_token = None

class StoryblokClient:
"""
Class that handles the storyblok client credentials as a global state.
Useful in this script context but careful to not re-use (or import)
this where the global state might affect your application.
"""

_storyblok_space_id: str
_storyblok_personal_access_token: str
_storyblok_base_url: str

REGION_TO_BASE_URLS = {
'eu': 'https://mapi.storyblok.com',
'us': 'https://api-us.storyblok.com',
'ca': 'https://api-ca.storyblok.com',
'au': 'https://api-ap.storyblok.com',
'cn': 'https://app.storyblokchina.cn'
}

def init_storyblok_client(space_id, token):
# TODO: make this cleaner without changing too much code
global _storyblok_space_id
global _storyblok_personal_access_token

_storyblok_space_id = space_id
_storyblok_personal_access_token = token


def request(method, path, params=None, **kwargs):
BASE_URL = 'https://mapi.storyblok.com'

return requests.request(
method,
f'{BASE_URL}/v1/spaces/{_storyblok_space_id}{path}',
headers={
'Authorization': _storyblok_personal_access_token,
},
params=params,
**kwargs,
)
DEFAULT_REGION = 'eu'

@classmethod
def init_client(cls, space_id, token, region):
if (
cls._storyblok_space_id
or cls._storyblok_personal_access_token
or cls._storyblok_base_url
):
raise RuntimeError("StoryblokClient already initialized")

cls.storyblok_space_id = space_id
cls.storyblok_personal_access_token = token
cls.storyblok_base_url = cls.REGION_TO_BASE_URLS[region]

@classmethod
def request(cls, method, path, params=None, **kwargs):
return requests.request(
method,
f'{cls.storyblok_base_url}/v1/spaces/{cls.storyblok_space_id}{path}',
headers={
'Authorization': cls.storyblok_personal_access_token,
},
params=params,
**kwargs,
)


def ensure_cache_dir_exists(cache_directory):
Expand All @@ -59,7 +80,7 @@ def save_json(file_path, data):
def download_asset(asset_url, target_file_path, continue_download_on_failure):
print(f'Downloading asset {asset_url!r} into {target_file_path!r}')

response = request('GET', asset_url, stream=True)
response = StoryblokClient.request('GET', asset_url, stream=True)

if not response.ok:
msg = f'Cannot download asset {asset_url}, got status code {response.status_code}'
Expand Down Expand Up @@ -90,7 +111,7 @@ def get_all_paginated(path, item_name, params={}):
'page': page,
}

response = request(
response = StoryblokClient.request(
'GET',
path,
params=params
Expand All @@ -117,9 +138,8 @@ def get_all_paginated(path, item_name, params={}):


def is_asset_in_use(asset):
file_path = asset['filename'].replace('https://s3.amazonaws.com/a.storyblok.com', '')

response = request(
file_path = asset['filename'].split('.storyblok.com', 1)[1]
response = StoryblokClient.request(
'GET',
'/stories',
params={
Expand All @@ -132,7 +152,6 @@ def is_asset_in_use(asset):
response.raise_for_status()

stories = response.json()['stories']

return len(stories) != 0


Expand All @@ -141,6 +160,32 @@ def _main():
description='storyblok-assets-cleanup an utility to delete unused assets.'
)

parser.add_argument(
'--token',
type=str,
default=getenv('STORYBLOK_PERSONAL_ACCESS_TOKEN'),
required=getenv('STORYBLOK_PERSONAL_ACCESS_TOKEN') is None,
help=(
'Storyblok personal access token, '
'alternatively use the env var STORYBLOK_PERSONAL_ACCESS_TOKEN.'
),
)
parser.add_argument(
'--space-id',
type=str,
default=getenv('STORYBLOK_SPACE_ID'),
required=getenv('STORYBLOK_SPACE_ID') is None,
help=(
'Storyblok space ID, alternatively use the env var STORYBLOK_SPACE_ID.'
),
)
parser.add_argument(
'--region',
type=str,
default=StoryblokClient.DEFAULT_REGION,
choices=list(StoryblokClient.REGION_TO_BASE_URLS.keys()),
help='Storyblok region (default: EU)'
)
parser.add_argument(
'--delete',
action=argparse.BooleanOptionalAction,
Expand All @@ -153,7 +198,16 @@ def _main():
action=argparse.BooleanOptionalAction,
type=bool,
default=True,
help='If we should backup assets (to ./assets_backup/<SPACE_ID>), defaults to true.',
help=(
'If we should backup assets (to the directory specified in `--backup-directory`), '
'defaults to true.'
),
)
parser.add_argument(
'--backup-directory',
type=str,
default='assets_backup',
help='Backup directory, defaults to ./assets_backup.',
)
parser.add_argument(
'--cache',
Expand All @@ -164,33 +218,19 @@ def _main():
'If we should use cache the assets index. Defaults to True (recommended).'
),
)
parser.add_argument(
'--cache-directory',
type=str,
default='cache',
help='Cache directory, defaults to ./cache.',
)
parser.add_argument(
'--continue-download-on-failure',
action=argparse.BooleanOptionalAction,
type=bool,
default=True,
help='If we should continue if the download of an asset fails. Defaults to true.',
)

parser.add_argument(
'--space-id',
type=str,
default=getenv('STORYBLOK_SPACE_ID'),
required=getenv('STORYBLOK_SPACE_ID') is None,
help=(
'Storyblok space ID, alternatively use the env var STORYBLOK_SPACE_ID.'
),
)
parser.add_argument(
'--token',
type=str,
default=getenv('STORYBLOK_PERSONAL_ACCESS_TOKEN'),
required=getenv('STORYBLOK_PERSONAL_ACCESS_TOKEN') is None,
help=(
'Storyblok personal access token, '
'alternatively use the env var STORYBLOK_PERSONAL_ACCESS_TOKEN.'
),
)
parser.add_argument(
'--blacklisted-folder-paths',
type=str,
Expand All @@ -212,22 +252,11 @@ def _main():
'Default to none/empty list.'
),
)
parser.add_argument(
'--cache-directory',
type=str,
default='cache',
help='Cache directory, defaults to ./cache.',
)
parser.add_argument(
'--backup-directory',
type=str,
default='assets_backup',
help='Backup directory, defaults to ./assets_backup.',
)

args = parser.parse_args()

init_storyblok_client(args.space_id, args.token)
StoryblokClient.init_client(args.space_id, args.token, args.region)

should_delete_images = args.delete
use_cache = args.cache
backup_assets = args.backup
Expand Down Expand Up @@ -281,7 +310,7 @@ def should_be_deleted(asset_path_name, filename):
print(f'Skipping {id} as it is in {asset_path_name}')
return False

if any(word in filename for word in blacklisted_asset_filename_words):
if any(word in filename for word in blacklisted_asset_filename_words if word):
print(f'Skipping {id} as it contains blacklisted words')
return False

Expand Down Expand Up @@ -400,7 +429,7 @@ def print_padded(outputs):
if should_delete_images:
print(f'Deleting asset {id}')

response = request(
response = StoryblokClient.request(
'DELETE',
f'/assets/{id}',
)
Expand Down

0 comments on commit 74596ba

Please sign in to comment.