diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b4d70c2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# File extensions +*.pyc +*.pyo +*~ +*.swp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a846902 --- /dev/null +++ b/LICENSE @@ -0,0 +1,69 @@ +========= + COPYING +========= + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public +License along with this program, in the file ``licenses/AGPLv3.txt``. +If not, see . + + +Translation files located under ``mediagoblin/i18n/`` directory tree +are free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at +your option) any later version. + +You should have received a copy of the GNU Affero General Public +License along with this program, in the file ``licenses/AGPLv3.txt``. +If not, see . + + +JavaScript files located in the ``mediagoblin/`` directory tree +are free software: you can redistribute and/or modify them under the +terms of the GNU Affero General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at +your option) any later version. + +You should have received a copy of the GNU Lesser General Public +License along with this program, in the file ``licenses/LGPLv3.txt``. +If not, see . + + +Documentation files located in the ``docs/`` directory tree and all +original documentation theme CSS and assets (including image files) +are released under a CC0 license. To the extent possible under law, +the author(s) have dedicated all copyright and related and neighboring +rights to these files to the public domain worldwide. These files are +distributed without any warranty. + +You should have received a copy of the CC0 license in the file +``licenses/CC0_1.0.txt``. If not, see +. + + +CSS, images and video located in the ``mediagoblin/`` directory tree are +released under a CC0 license. To the extent possible under law, the author(s) +have dedicated all copyright and related and neighboring rights to these +files to the public domain worldwide. These files are distributed without +any warranty. + +You should have received a copy of the CC0 license in the file +``licenses/CC0_1.0.txt``. If not, see +. + + +Additional library software has been made available in the ``extlib/`` +directory. All of it is Free Software and can be distributed under +liberal terms, but those terms may differ in detail from the AGPL's +particulars. See each package's license file in the extlib directory +for additional terms. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..873b20e --- /dev/null +++ b/README.rst @@ -0,0 +1,45 @@ +============================================ + gmg_localfiles, plugin for GNU MediaGoblin +============================================ + +Plugin for importing files from your filesystem without duplication. + +This plugin lets you have all your original files in one folder on your file +system, and it will stop MediaGoblin from copying those files to its own +locations. + +It will try to make mediagoblin not touch/ruin your files (no guarantees!), but +it will make a `mg_cache` folder in the directory. + +Example setup in `mediagoblin.ini`:: + + [storage:queuestore] + base_dir = /srv/media/Pictures + storage_class = gmg_localfiles.storage:PersistentFileStorage + + [storage:publicstore] + base_dir = /srv/media/Pictures + base_url = /mgoblin_media/ + storage_class = gmg_localfiles.storage:PersistentFileStorage + + [plugins] + [[gmg_localfiles]] + +You will also need to serve the files, so in `paste.ini`:: + + [app:publicstore_serve] + use = egg:Paste#static + document_root = %(here)s/user_dev/media/public/ + +-------------- + Installation +-------------- + +Put gmg_localfiles somewhere on your Python path. You might even just put it +inside the MediaGoblin folder if you want to be done with it quickly ;-) + +--------- + Running +--------- + +Go into the `gmg_localfiles` folder and run `python import_files.py`. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..ee70eda --- /dev/null +++ b/__init__.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# GMG localfiles plugin -- local file import +# Copyright (C) 2012 Odin Hørthe Omdal +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys +import logging + +from mediagoblin.tools.pluginapi import get_config +from mediagoblin import processing + +_log = logging.getLogger(__name__) + + +# Monkeypatch create_pub_filepath to not clean the original files, and to +# rather use queued_media_file instead of hardcoded path. +from storage import _is_cachefile +from mediagoblin.storage import clean_listy_filepath +def monkey_create_pub_filepath(entry, filename): + if _is_cachefile(filename): + filepath = clean_listy_filepath(entry.queued_media_file) + else: + filepath = list(entry.queued_media_file) + + filepath[-1] = filename + return filepath +processing.create_pub_filepath = monkey_create_pub_filepath + + +class PreservingFilenameBuilder(processing.FilenameBuilder): + def __init__(self, path): + """Initialize a builder from an original file path.""" + self.dirpath, self.basename = os.path.split(path) + self.basename, self.ext = os.path.splitext(self.basename) + + def fill(self, fmtstr): + basename_len = (self.MAX_FILENAME_LENGTH - + len(fmtstr.format(basename='', ext=self.ext))) + ext = self.ext + if _is_cachefile(fmtstr): + ext = ext.lower() + return fmtstr.format(basename=self.basename[:basename_len], + ext=ext) +processing.FilenameBuilder = PreservingFilenameBuilder + + +def setup_plugin(): + _log.info('LocalFiles plugin set up!') + config = get_config('gmg_localfiles') + if not config: + _log.info('There is no configuration set.') + sys.exit(1) + +hooks = { + 'setup': setup_plugin + } diff --git a/import_files.py b/import_files.py new file mode 100644 index 0000000..a7da57d --- /dev/null +++ b/import_files.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GMG localfiles plugin -- local file import +# Copyright (C) 2012 Odin Hørthe Omdal +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +# This is here early because of a race +from mediagoblin.app import MediaGoblinApp +if __name__ == "__main__": + config_file = '/home/odin/src/mediagoblin/mediagoblin.ini' + mg = MediaGoblinApp(config_file, setup_celery=True) + from mediagoblin import mg_globals + + from mediagoblin.init.celery import setup_celery_app + setup_celery_app(mg_globals.app_config, \ + mg_globals.global_config, force_celery_always_eager=True) + +import os +import uuid + +from celery import registry + +from mediagoblin.tools.text import convert_to_tag_list_of_dicts +from mediagoblin.storage import clean_listy_filepath +from mediagoblin.processing import mark_entry_failed +from mediagoblin.processing.task import ProcessMedia +from mediagoblin.media_types import sniff_media, \ + InvalidFileType, FileTypeNotSupported + + +class MockMedia(): + filename = "" + stream = None + def __init__(self, filename, stream): + self.filename = filename + self.stream = stream + + +class ImportCommand(object): + #args = '' + help = 'Find new photos and add to database' + + def __init__(self, db, base_dir, **kwargs): + self.db = db + self.base_dir = base_dir + + def handle(self): + #Photo.objects.all().delete() + + os.chdir(self.base_dir) + + for top, dirs, files in os.walk(u'.'): + # Skip hidden folders + if '/.' in top: + continue + # Skip cache folders + if '_cache' in top: + print "cache skip", top + continue + if top == ".": + top = "" + + #folder, new_folder = Folder.objects.select_related("photos") \ + # .get_or_create(path=os.path.normpath(top) + "/", + # defaults={'name': os.path.basename(top)}) + folder_path = os.path.normpath(top) + try: + cleaned_top = "/".join(clean_listy_filepath(folder_path.split("/"))) + except Exception: + cleaned_top = top + new_folder = not os.path.exists(os.path.join("mg_cache", cleaned_top)) + + if not new_folder: + print u"Skipping folder {0}".format(folder_path).encode("utf-8") + continue + new_files = [os.path.splitext(i)[0] for i in files] + new_files.sort(reverse=True) + + for new_filename in new_files: + file_url = os.path.join(folder_path, new_filename) + + # More than one file with the same name but different + # extension? + exts = [os.path.splitext(f)[1] for f in files if new_filename + in f] + + assert len(exts) > 0, "Couldn't find file extension for %s" % file_url + + # If there exists NEF file, prefer that as canonical file + if '.nef' in exts and os.path.exists(file_url + '.nef'): + f = file_url + '.nef' + elif '.NEF' in exts and os.path.exists(file_url + '.NEF'): + f = file_url + '.NEF' + else: + f = file_url + exts[0] + + try: + m = MockMedia(filename=f, stream=open(f, "r")) + self.import_file(m) + except Exception as e: + print u"file: {0} exception: {1}".format(f, e).encode('utf-8') + continue + + + def import_file(self, media): + try: + media_type, media_manager = sniff_media(media) + except (InvalidFileType, FileTypeNotSupported) as e: + print u"File error {0}: {1}".format(media.filename, repr(e)).encode("utf-8") + return + entry = self.db.MediaEntry() + entry.media_type = unicode(media_type) + entry.title = unicode(os.path.splitext(media.filename)[0]) + + entry.uploader = 1 + # Process the user's folksonomy "tags" + entry.tags = convert_to_tag_list_of_dicts("") + # Generate a slug from the title + entry.generate_slug() + + task_id = unicode(uuid.uuid4()) + + entry.queued_media_file = media.filename.split("/") + entry.queued_task_id = task_id + + entry.save(validate=True) + + process_media = registry.tasks[ProcessMedia.name] + try: + process_media.apply_async( [unicode(entry.id)], {}, task_id=task_id) + except BaseException as exc: + mark_entry_failed(entry.id, exc) + raise + + +if __name__ == "__main__": + from mediagoblin import mg_globals + ic = ImportCommand(mg.db, mg_globals.global_config['storage:publicstore']['base_dir']) + ic.handle() diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..1bbd5f4 --- /dev/null +++ b/storage.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +# GMG localfiles plugin -- local file import +# Copyright (C) 2012 Odin Hørthe Omdal +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.storage import ( + clean_listy_filepath, + NoWebServing) +from mediagoblin.storage.filestorage import BasicFileStorage + +import os +import shutil +import logging +import urlparse + +_log = logging.getLogger(__name__) + +def _is_cachefile(filepath): + if not isinstance(filepath, basestring): + filepath = filepath[-1] + return any(k in filepath for k in ['thumbnail', 'medium']) + +class PersistentFileStorage(BasicFileStorage): + """ + Local filesystem implementation of storage API that doesn't delete files + """ + + def _resolve_filepath(self, filepath, force_cache=False): + """ + Transform the given filepath into a local filesystem filepath. + """ + if _is_cachefile(filepath) or force_cache: + filepath = clean_listy_filepath(list(filepath)) + filepath.insert(0, "mg_cache") + + return os.path.join( + self.base_dir, *filepath) + + def file_url(self, filepath): + if not self.base_url: + raise NoWebServing( + "base_url not set, cannot provide file urls") + + if _is_cachefile(filepath): + filepath = clean_listy_filepath(list(filepath)) + filepath.insert(0, "mg_cache") + + return urlparse.urljoin( + self.base_url, + '/'.join(filepath)) + + def get_file(self, filepath, mode='r'): + if _is_cachefile(filepath): + return super(PersistentFileStorage, self).get_file(filepath, mode) + if not os.path.exists(self._resolve_filepath(filepath)): + return PersistentStorageObjectWrapper(None, self._resolve_filepath(filepath)) + + mode = mode.replace("w", "r") + # Grab and return the file in the mode specified + return PersistentStorageObjectWrapper( + open(self._resolve_filepath(filepath), mode)) + + def delete_file(self, filepath): + #os.remove(self._resolve_filepath(filepath)) + _log.info(u'Not removing {0} as requested.'.format(self._resolve_filepath(filepath))) + + def copy_local_to_storage(self, filename, filepath): + """ + Copy this file from locally to the storage system. + """ + # Make directories if necessary + if len(filepath) > 1: + directory = self._resolve_filepath(filepath[:-1], force_cache=True) + if not os.path.exists(directory): + os.makedirs(directory) + + shutil.copy(filename, self.get_local_path(filepath)) + +class PersistentStorageObjectWrapper(): + def __init__(self, storage_object, name=None, *args, **kwargs): + self.storage_object = storage_object + self.name = name + if storage_object: + self.name = storage_object.name + + def read(self, *args, **kwargs): + _log.debug(u'Reading {0}'.format( + self.name).encode("utf-8")) + return self.storage_object.read(*args, **kwargs) + + def write(self, data, *args, **kwargs): + _log.debug(u'Not writing {0}'.format( + self.name).encode("utf-8")) + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close()