diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..8dde98b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +# Test coverage configuration +# Usage: +# python -m coverage erase # clears previous data if any +# python -m coverage run scripts/run-tests +# python -m coverage report # prints to stdout +# python -m coverage html # create ./htmlcov/*.html including annotated source +# open htmlcov/index.html +[run] +source = + backend + scripts + +[report] +# Ignore missing source files, i.e. fake tornado template-generated "files" +ignore_errors = true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e27b707 --- /dev/null +++ b/.gitignore @@ -0,0 +1,81 @@ +syntax: glob +*~ +*.pyc +*.orig +*.pb.cc +*.pb.h +*.trace +*.tmp +.*.swp +.libs/ +.deps/ +#* +.#* +*_pkg.py +.fuse_* +.DS_Store +local/ +proto_gen/ +thrift_gen/ + +.idea/workspace.xml +.idea/tasks.xml + +background/cockroach/cockroach +background/cockroach/cockroach_bench +background/cockroach/*_test +background/cockroach/build_config.mk + +clients/ios/.idea/ +clients/ios/DerivedData/ +xcuserdata/ + +*.xcodeproj/ +*.xccheckout + +eclipse/backend-project/.settings +.tox +.vagrant +viewfinder.egg-info/* +hg_revision.txt +.coverage +htmlcov +build + +backend/resources/static/.webassets-cache +backend/resources/static/gen + +clients/android/experimental/hello/.idea/ +clients/android/experimental/hello/bin/ +clients/android/experimental/hello/gen/ +clients/android/experimental/hello/hello.iml +clients/android/experimental/hello/libs/*/*.so +clients/android/experimental/hello/local.properties +clients/android/experimental/hello/proguard-project.txt + +clients/android/bin/ +clients/android/gen/ +clients/android/out/ +clients/android/libs/*/*.so +clients/android/libs/*/gdb.setup +clients/android/libs/*/gdbserver +clients/android/local.properties +clients/android/proguard-project.txt +clients/android/res/xml/properties.xml + +clients/ios/testing/data/cookies.txt +clients/ios/testing/.project +clients/ios/testing/.pydevproject +clients/ios/testing/js/.project +clients/ios/testing/js/.settings +clients/ios/testing/js/control/* +clients/ios/testing/html/summary_results.html +clients/ios/testing/results/archive/ +clients/ios/testing/results/current/ +clients/ios/testing/snapshots/Media.backup +clients/ios/testing/snapshots/Library/AddressBook.backup + +# Ignore gyp-generated android makefiles. We can't ignore all .mk files yet, we still have some hand-crafted ones. +*.mk +clients/shared/*.mk +clients/android/jni/viewfinder.target.mk diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..04a9f83 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party"] + path = third_party + url = ../third_party.git diff --git a/.hgignore b/.hgignore new file mode 100644 index 0000000..2a27569 --- /dev/null +++ b/.hgignore @@ -0,0 +1,85 @@ +syntax: glob +*~ +*.pyc +*.orig +*.pb.cc +*.pb.h +*.trace +*.tmp +.*.swp +.libs/ +.deps/ +#* +.#* +*_pkg.py +.fuse_* +.DS_Store +local/ +proto_gen/ +thrift_gen/ + +.idea/workspace.xml +.idea/tasks.xml + +background/cockroach/cockroach +background/cockroach/cockroach_bench +background/cockroach/*_test +background/cockroach/build_config.mk + +clients/ios/.idea/ +clients/ios/DerivedData/ +xcuserdata/ + +*.xcodeproj/ + +eclipse/backend-project/.settings +.tox +.vagrant +viewfinder.egg-info/* +hg_revision.txt +.coverage +htmlcov +build + +backend/resources/static/.webassets-cache +backend/resources/static/gen + +clients/android/experimental/hello/.idea/ +clients/android/experimental/hello/bin/ +clients/android/experimental/hello/gen/ +clients/android/experimental/hello/hello.iml +clients/android/experimental/hello/libs/*/*.so +clients/android/experimental/hello/local.properties +clients/android/experimental/hello/proguard-project.txt + +clients/android/bin/ +clients/android/gen/ +clients/android/out/ +clients/android/libs/*/*.so +clients/android/libs/*/gdb.setup +clients/android/libs/*/gdbserver +clients/android/local.properties +clients/android/proguard-project.txt +clients/android/res/xml/properties.xml + +clients/ios/testing/data/cookies.txt +clients/ios/testing/.project +clients/ios/testing/.pydevproject +clients/ios/testing/js/.project +clients/ios/testing/js/.settings +clients/ios/testing/js/control/* +clients/ios/testing/html/summary_results.html +clients/ios/testing/results/archive/ +clients/ios/testing/results/current/ +clients/ios/testing/snapshots/Media.backup +clients/ios/testing/snapshots/Library/AddressBook.backup + +# Ignore gyp-generated android makefiles. We can't ignore all .mk files yet, we still have some hand-crafted ones. +*.mk +clients/shared/*.mk +clients/android/jni/viewfinder.target.mk + +# The fetch_logs script puts its output in ./logs, but we also have code in backend/logs. +# We must use regexp syntax to match one and not the other. +syntax: regexp +^logs diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..452d1ca --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +Viewfinder +========== + +Setup +----- + +We use subrepositories, so after cloning (or pulling any change that includes a change to +the subrepository), you must run + + $ git submodule update --init + +Many of the following scripts require certain `PATH` entries or other environment variables. +Set them up with the following (this is intended to be run from `.bashrc` or other shell +initialization scripts; if you do not install it there you will need to repeat this command +in each new terminal): + + $ source scripts/viewfinder.bash + +Server +------ + +To install dependencies (into `~/envs/vf-dev`), run + + $ update-environment + +To run unit tests: + + $ run-tests + +TODO: add ssl certificates and whatever else local-viewfinder needs, and document running it. + +iOS client +---------- + +Our Xcode project files are generated with `gyp`. After checking out the code +(and after any pull in which a `.gyp` file changed), run + + $ generate-projects.sh + +Open the workspace containing the project, *not* the generated project itself: + + $ open clients/ios/ViewfinderWorkspace.xcworkspace + +Android client +-------------- + +The android client is **unfinished**. To build it, run + + $ generate-projects-android.sh + $ vf-android.sh build diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..47d18d5 --- /dev/null +++ b/__init__.py @@ -0,0 +1,89 @@ +import logging +import os +import subprocess + +def exec_cmd(cmd): + logging.info("%s", ' '.join(cmd)) + + try: + p = subprocess.Popen(cmd, bufsize=-1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = p.communicate() + except Exception: + logging.error("%s", err) + raise + if p.returncode: + raise Exception('%s\n%s%s' % (' '.join(cmd), out, err)) + +def ensure_dir(dir): + try: + os.makedirs(dir) + except: + pass + +def get_path(filename): + return os.path.join(__path__[0], filename) + +def get_stamp(source, gen_dir, ext): + return os.path.join(gen_dir, os.path.splitext(source)[0] + ext) + +def refresh(source, stamp, cmd): + try: + if os.path.getmtime(source) == os.path.getmtime(stamp): + return + except: + pass + + try: + exec_cmd(cmd) + ensure_dir(os.path.dirname(stamp)) + open(stamp, 'a').close() # ensure the stamp file exists + os.utime(stamp, (os.path.getatime(source), os.path.getmtime(source))) + except: + logging.exception('unable to generate files') + try: + os.unlink(stamp) + except: + pass + +def proto_gen_cmd(source, gen_type, gen_dir): + ensure_dir(gen_dir) + if gen_type == 'py': + return ['protoc', source, '--python_out=%s' % os.path.dirname(source), + '-I%s' % os.path.dirname(source)] + +def proto_gen_py(source, gen_dir): + stamp = get_path(get_stamp(source, gen_dir, '.py_stamp')) + output_dir = os.path.join(gen_dir, os.path.dirname(source)) + source = get_path(source) + refresh(source, stamp, proto_gen_cmd(source, 'py', output_dir)) + +def thrift_gen_cmd(source, gen_type, gen_dir): + ensure_dir(gen_dir) + return ['thrift', '-gen', gen_type, '-out', gen_dir, source] + +def thrift_gen_py(source, gen_dir): + stamp = get_path(get_stamp(source, gen_dir, '.py_stamp')) + source = get_path(source) + refresh(source, get_path(stamp), + thrift_gen_cmd(source, 'py', get_path(gen_dir))) + +def thrift_gen_js(source, gen_dir): + stamp = get_path(get_stamp(source, gen_dir, '.js_stamp')) + gen_dir = os.path.join(gen_dir, os.path.dirname(source)) + source = get_path(source) + refresh(source, stamp, + thrift_gen_cmd(source, 'js:jquery', get_path(gen_dir))) + +# Automatically (re-)generate any thrift sources when the viewfinder module is +# first imported. +thrift_gen_dir = 'thrift_gen' +#thrift_gen_py('hbase/Hbase.thrift', thrift_gen_dir) +#thrift_gen_py('backend/www/operation.thrift', thrift_gen_dir) +#thrift_gen_js('backend/www/operation.thrift', thrift_gen_dir) + +# Generate protocol buffer sources. +proto_gen_dir = 'proto_gen' +#proto_gen_py('backend/proto/server.proto', proto_gen_dir) + diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..1dbdb47 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1,2 @@ +assert __name__ == 'viewfinder.backend', \ + 'import the backend module as viewfinder.backend, not as a top-level module.' diff --git a/backend/base/__init__.py b/backend/base/__init__.py new file mode 100644 index 0000000..9970bb3 --- /dev/null +++ b/backend/base/__init__.py @@ -0,0 +1,10 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Add functionality to dump stack trace on SIGUSR1 for all python +processes. +""" + +import signal +import traceback + +signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack)) diff --git a/backend/base/ami_metadata.py b/backend/base/ami_metadata.py new file mode 100755 index 0000000..f0dde8a --- /dev/null +++ b/backend/base/ami_metadata.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""An interface to access Amazon Machine Instance (AMI) metadata. + +The Metadata class provides a convenient interface to AMI metadata and +userdata. Metadata is configured dynamically and for each instance +individually; it contains values such as local and public IP +addresses, AMI instance and type information, launch index, and +security groups. Userdata is specified statically at instance launch +time and all instances receive the same userdata contents. + +Internally, uses the tornado asynchronous http client to retrieve AMI +instance metadata from the static IP address: +http://169.254.169.254/latest/{meta-data,user-data}. + + Metadata: retrieves commonly used metadata and userdata and provides + an interface for retrieving additional metadata. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import sys +from functools import partial +from tornado import httpclient, options, web, ioloop +from viewfinder.backend.base import util + + +options.define("mock", default=False, + help="True to create a mock server for testing") + +_metadata = dict() +"""Instance-specific AMI metadata.""" + +def GetAMIMetadata(): + return _metadata + +def SetAMIMetadata(metadata): + global _metadata + _metadata = metadata + +class Metadata(object): + """Data object that provides a convenient interface to the Amazon AMI + metdata and userdata. On instantiation, a default set of most + commonly used metadata is queried asynchronously by default. If + callback is specified, then it is invoked upon completion of this + first set of queries. The default set contains: + + ami-id, ami-launch-index, hostname, instance-id, instance-type, + local-hostname, local-ipv4, public-hostname, public-ipv4 + """ + _QUERY_IP = "169.254.169.254" + _QUERY_VERSION = "latest" + + def __init__(self, callback=None, query_ip=_QUERY_IP, query_version=_QUERY_VERSION): + """Creates a `Metadata`. + + If `callback` is specified, launches async retrieval of commonly + used metadata values and the userdata and invokes `callback` upon + completion. Callback is invoked with a dictionary containing the + common metadata and the userdata. + + :arg callback: invoked when default metadata is available + :arg query_ip: IP address to query for metadata; + default 169.254.169.254 + :arg query_version: Version of metadata; default 'latest' + """ + self._query_ip = query_ip + self._query_version = query_version + if callback: + self._FetchCommonMetadata(callback) + + def FetchMetadata(self, paths, callback): + """Asynchronously fetches metadata for the specified path(s) and + on completion invokes the callback with the retrieved metadata + value. 'paths' can be iterable over multiple metadata to fetch; + if not, adds it to a list. + """ + metadata = {} + + def _OnFetch(path, callback, response): + if response.code == 200: + metadata[path] = response.body.strip() + else: + logging.error("error fetching '%s': %s", path, response.error) + callback() + + if type(paths) in (unicode, str): + paths = [paths] + + with util.Barrier(partial(callback, metadata)) as b: + for path in paths: + logging.info('fetching metadata from %s' % self._GetQueryURL(path)) + http_client = httpclient.AsyncHTTPClient() + http_client.fetch(self._GetQueryURL(path), partial(_OnFetch, path, b.Callback()), + connect_timeout=1, request_timeout=5) + + def _GetQueryURL(self, path): + """Returns a query URL for instance metadata using the specified path. + """ + return "http://{0}/{1}/{2}".format( + self._query_ip, self._query_version, path) + + def _FetchCommonMetadata(self, callback): + """Fetches common metadata values and compiles the results into + a dictionary, which is passed to the callback on completion. + + NOTE: the AWS metadata server has some sort of internal rate-limiting + for this data and will return 404 errors if too many are done + in parallel. So, we fetch them serially. + """ + paths = [ "meta-data/hostname", "meta-data/instance-id", "user-data/passphrase" ] + self.FetchMetadata(paths, callback) + + +def main(): + """Creates a Metadata object, fetches the default dictionary of + metadata values, and prints them. + + If --mock was specified on the command line, creates an http server + for testing. + """ + query_ip = Metadata._QUERY_IP + + # If a mock server was requested for testing, start it here. + options.parse_command_line() + if options.options.mock: + from tornado import testing + port = testing.get_unused_port() + class Handler(web.RequestHandler): + def get(self, path): + self.write(path.split("/")[-1]) + application = web.Application([ (r"/(.*)", Handler), ]) + application.listen(port) + query_ip = "localhost:{0}".format(port) + + def _MetadataCallback(metadata): + print metadata + ioloop.IOLoop.current().stop() + + Metadata(callback=_MetadataCallback, query_ip=query_ip) + ioloop.IOLoop.current().start() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/base/base64hex.py b/backend/base/base64hex.py new file mode 100644 index 0000000..59f8c53 --- /dev/null +++ b/backend/base/base64hex.py @@ -0,0 +1,70 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. +"""Order-preserving variant of base64. + +Uses the URL-safe 64 letter alphabet [0-9A-Za-z-_] to base-64 +encode/decode binary strings. However, the values assigned to +each alphanumeric character properly preserve the sort ordering +of the original byte strings. + +Based on the "Base 32 Encoding with Extended Hex Alphabet", as +described in RFC 4648, which preserves the bitwise sort order of +the original binary string. + +http://tools.ietf.org/html/rfc4648 + + B64HexEncode: encodes bytes to b64 hex encoding + B64HexDecode: decodes bytes from b64 hex encoding +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'ben@emailscrubbed.com (Ben Darnell)'] + +import base64 +import re +import string +from tornado.escape import utf8 + +_std_alphabet = string.uppercase + string.lowercase + string.digits + '+/' +_b64hex_alphabet = '-' + string.digits + string.uppercase + '_' + string.lowercase + +assert sorted(_b64hex_alphabet) == list(_b64hex_alphabet) + +_std_to_b64hex = string.maketrans(_std_alphabet, _b64hex_alphabet) +_b64hex_to_std = string.maketrans(_b64hex_alphabet, _std_alphabet) + +_valid_char_re = re.compile('^[a-zA-Z0-9_-]*={0,3}$') + +def B64HexEncode(s, padding=True): + """Encode a string using Base64 with extended hex alphabet. + + s is the string to encode. The encoded string is returned. + """ + encoded = base64.b64encode(s) + translated = encoded.translate(_std_to_b64hex) + if padding: + return translated + else: + return translated.rstrip('=') + +_PAD_LEN = 4 + +def B64HexDecode(s, padding=True): + """Decode a Base64 encoded string. + + The decoded string is returned. A TypeError is raised if s is + incorrectly padded or if there are non-alphabet characters present + in the string. + """ + s = utf8(s) + # In python2.7 b64decode doesn't do the validation the docs say it does + # http://bugs.python.org/issue1466065 + if not _valid_char_re.match(s): + raise TypeError("Invalid characters") + pad_needed = len(s) % _PAD_LEN + if pad_needed: + if padding: + raise TypeError("Invalid padding") + else: + s += '=' * pad_needed + translated = s.translate(_b64hex_to_std) + return base64.b64decode(translated) diff --git a/backend/base/base_options.py b/backend/base/base_options.py new file mode 100644 index 0000000..d7e31b6 --- /dev/null +++ b/backend/base/base_options.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# +# Copyright 2012 Viewfinder Inc. All Rights reserved. +"""Command-line options used by base or higher layers. + +These options are considered base in that they may be imported by any layer +at or above the base layer. +""" + +import os +import viewfinder +from tornado import options + +options.define('secrets_dir', os.path.join(viewfinder.__path__[0], 'secrets'), + help='directory containing secrets files') + +options.define('user_secrets_dir', os.path.join(os.path.expanduser('~/.secrets')), + help='directory containing secrets files for the running user') + +options.define('passphrase', None, help='the passphrase for decoding secrets') + +options.define('passphrase_file', None, help='file containing the passphrase') + +options.define('domain', + default='viewfinder.co', + help='service domain (for redirects, keys, etc.)') + +options.define('www_label', + default='www', + help='The label to use for the prod_host domain (for redirects, etc.)') + +options.define('short_domain', + default='vfnd.co', + help='domain used in some short URLs for SMS, etc.') + +options.define('devbox', + default=False, + help='start in dev/desktop mode outside of EC2') + +options.define('is_staging', + type=bool, + default=None, + help='start in staging mode outside of EC2') + +options.define('vf_temp_dir', default=None, + help='directory for viewfinder temp files if running as devbox') diff --git a/backend/base/client_version.py b/backend/base/client_version.py new file mode 100644 index 0000000..52df1da --- /dev/null +++ b/backend/base/client_version.py @@ -0,0 +1,53 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Client versioning handler. + +Must be kept up to date with the versioning logic in the client: +//viewfinder/client/ios/Source/Utils.mm::AppVersion + +TODO(marc): should we exclude 'adhoc' and 'dev' when comparing versions? +TODO(marc): handle android versioning once we have it. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +class ClientVersion(object): + def __init__(self, version): + self.version = version + self.components = version.split('.') if version is not None else None + + def IsValid(self): + if not self.version: + return False + if not self.components: + return False + # Require at least one dot in the name. + if len(self.components) < 2: + return False + + # TODO(marc): we may want additional logic here (eg: w.x.y.z). + return True + + def IsDev(self): + return self.version.endswith('.dev') + + def IsTestFlight(self): + return self.version.endswith('.adhoc') + + def IsAppStore(self): + return not self.IsDev() and not self.IsTestFlight() + + def LT(self, version): + return cmp(self.components, version.split('.')) < 0 + + def LE(self, version): + return cmp(self.components, version.split('.')) <= 0 + + def EQ(self, version): + return cmp(self.components, version.split('.')) == 0 + + def GT(self, version): + return cmp(self.components, version.split('.')) > 0 + + def GE(self, version): + return cmp(self.components, version.split('.')) >= 0 diff --git a/backend/base/constants.py b/backend/base/constants.py new file mode 100644 index 0000000..d6fba7c --- /dev/null +++ b/backend/base/constants.py @@ -0,0 +1,21 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder constants. + +Constants used throughout the code base. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + + +SECONDS_PER_MINUTE = 60 +"""Number of seconds in a minute.""" + +SECONDS_PER_HOUR = 60 * 60 +"""Number of seconds in an hour.""" + +SECONDS_PER_DAY = 60 * 60 * 24 +"""Number of seconds in a day.""" + +SECONDS_PER_WEEK = SECONDS_PER_DAY * 7 +"""Number of seconds in a week.""" diff --git a/backend/base/context_local.py b/backend/base/context_local.py new file mode 100644 index 0000000..ab450e4 --- /dev/null +++ b/backend/base/context_local.py @@ -0,0 +1,74 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" Module to support storing arbitrary contextual information using +tornado's StackContext class. + + * ContextLocal: base class which implements context-local instance management. + * ViewfinderContext: context-local class for storing context during a viewfinder request. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import threading + +class _ContextLocalManager(threading.local): + """Extension of threading.local which ensures that the 'current' attribute + defaults to an empty dict for each thread. + """ + def __init__(self): + self.current = dict() + + +class ContextLocal(object): + """Base class for objects which have a context-local instance. An instance + of a derived class can be pushed onto the persistent stack using a + StackContext object. The currently in-scope instance of a derived class + can be retrieved from the stack with the class method cls.current(). + This mimics the concept of a thread-local object, but the object is linked + to the persistent stack context provided by Tornado. + + Example: + + # Create a stack-aware context class + class MyContext(ContextLocal): + def __init__(self, val): + self.some_value = val + + # Push a new context onto the stack, and verify a value in it: + with StackContext(MyContext(val)): + assert MyContext.current().some_value == val + """ + _contexts = _ContextLocalManager() + _default_instance = None + + def __init__(self): + """Maintain stack of previous instances. This is a stack to support re-entry + of a context. + """ + self.__previous_instances = [] + + @classmethod + def current(cls): + """Retrieves the currently in-scope instance of context class cls, or a + default instance if no instance is currently in scope. + """ + current_value = cls._contexts.current.get(cls.__name__, None) + return current_value if current_value is not None else cls._default_instance + + def __enter__(self): + """Sets this instance to be the currently in-scope instance of its class.""" + cls = type(self) + self.__previous_instances.append(cls._contexts.current.get(cls.__name__, None)) + cls._contexts.current[cls.__name__] = self + + def __exit__(self, exc_type, exc_value, exc_traceback): + """Sets the currently in-scope instance of this class to its previous value.""" + cls = type(self) + cls._contexts.current[cls.__name__] = self.__previous_instances.pop() + + def __call__(self): + """StackContext takes a 'context factory' as a parameter, which is a callable + which should return a context object. By making an instance of this class return + itself when called, each instance becomes its own factory. + """ + return self diff --git a/backend/base/counters.py b/backend/base/counters.py new file mode 100644 index 0000000..2637d40 --- /dev/null +++ b/backend/base/counters.py @@ -0,0 +1,337 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" Module to support custom performance counters in python modules. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import time +from dotdict import DotDict +from collections import deque + + +class _CounterManager(DotDict): + """A CounterManager object is used as a central place for modules to register their + performance counters. The manager allows access to the counters utilizing a simple + namespace notation. + """ + def register(self, counter): + """Register a counter with the manager. Counters are organized + into namespaces using '.' as a separator. Examples of module names: + + # Valid counter names + my_counter + module.counters.another_counter + + Note that it is invalid for a counter's name to be the namespace of + another counter: + + # Invalid counter names, due to namespace conflict: + my_module.counter + my_module.counter.invalid + """ + cname = counter.name + if len(cname) == 0: + raise ValueError('Cannot register counter with a blank name.') + + # Verify that there are no namespace conflicts. + existing = self.get(cname, None) + if existing: + if isinstance(existing, DotDict): + raise KeyError('Cannot register counter with name %s because a namespace' + ' with the same name was previously registered.' % cname) + else: + raise KeyError('Cannot register counter with name %s because a counter' + ' with the same name was previously registered.' % cname) + + # Insert the actual counter. + try: + self[cname] = counter + except KeyError: + raise KeyError('Cannot register counter with name %s because a portion of its' + ' namespace conflicts with a previously registered counter.' % cname) + + +# Global instance of CounterManager. +counters = _CounterManager() + + +def define_total(name, description, manager=counters): + """Creates a performance counter which tracks some cumulative value over the + course of the program. The performance counter can be incremented using one + of several increment methods: + + # Define a new total counter in the module. + total_counter = counters.total('module.counters.total', 'Total count of events.') + + # ... + + total_counter.increment() # Increment by 1 + total_counter.increment(20) + total_counter.decrement() # Decrement by 1 + total_counter.decrement(20) + + When sampled using a Meter, this counter returns the total accumulated value of the + counter since the start of the program. + """ + counter = _TotalCounter(name, description) + manager.register(counter) + return counter + + +def define_delta(name, description, manager=counters): + """Creates a performance counter which tracks the accumulation of a value since the previous + sample of the counter, thus providing a delta of the underlying value of the counter. The + performance counter can be incremented using one of several increment methods: + + # Define a new delta counter in the module. + delta_counter = counters.delta('module.counters.delta', 'Count of new events.') + + # ... + + delta_counter.increment() # Increment by 1 + delta_counter.increment(20) + delta_counter.decrement() # Decrement by 1 + delta_counter.decrement(20) + + When sampled using a Meter, this counter returns the difference in the accumulated value + since the previous sample of the Meter. + """ + counter = _DeltaCounter(name, description) + manager.register(counter) + return counter + + +def define_rate(name, description, unit_seconds=1, manager=counters): + """Creates a performance counter which tracks some rate at which a value accumulates + over the course of the program. The counter has an optional 'unit_seconds' parameter + which determines the time unit associated with the value - the default is one second. + + The counter can be incremented using one of several increment methods: + + # Define a new rate counter in the module. + rate_counter = counters.rate('module.counters.rate', 'Accumulation per minute', unit_seconds=60) + + # ... + + rate_counter.increment() # Increment by 1 + rate_counter.increment(20) + rate_counter.decrement() # Decrement by 1 + rate_counter.decrement(20) + + When sampled using a Meter, this counter returns the average rate of change in the underlying value + per the given unit of time, taken over the time span since the previous sample of the Meter. + """ + counter = _RateCounter(name, description, unit_seconds) + manager.register(counter) + return counter + + +def define_average(name, description, manager=counters): + """Creates a performance counter which tracks the average value of a quantity which varies + for discrete occurrences of an event. An example would be the average time taken to complete + an operation, or the bytes transferred per operation. Unlike other counters, this counter + provides only a single method 'add()', which is called with the quantity for a single + occurrence of the event. The counter will essentially track the average of all numbers + passed to the 'add()' method. + + # Define a new average counter in the module. + avg_counter = counters.average('module.counters.average', 'Average bytes per request') + + # ... + + response = perform_some_request() + avg_counter.add(response.bytes_transfered) + + + When sampled using a Meter, this counter returns the average value of quantities passed + to 'add()' for all events since the previous sample of the Meter. + """ + counter = _AverageCounter(name, description) + manager.register(counter) + return counter + + +class _BaseCounter(object): + """ Basic counter object, which should not be directly instantiated. Implements + the common method 'get_sampler()', which returns a closure function which can be + called repeatedly to sample the counter. + """ + def __init__(self, name, description): + self.name = name + self.description = description + + def get_sampler(self): + """ Returns a closure function which can be called repeatedly to sample the counter. + The use of a closure function ensures that multiple Meters can be used simultaneously + without interference. + """ + last_sample = [self._raw_sample()] + def sampler_func(): + old_sample = last_sample[0] + last_sample[0] = self._raw_sample() + return self._computed_sample(old_sample, last_sample[0]) + return sampler_func + + def _raw_sample(self): + """Returns a raw sample for the counter, which represents the value of internal + counters at the moment the sample is taken. Two raw samples will be used inside + of _computed_sample() to return a value from the counter with proper units. + """ + raise NotImplementedError('_raw_sample() must be implemented in a subclass.') + + def _computed_sample(self, s1, s2): + """Using two raw samples taken previously, creates a sample in units + which are appropriate to the specific type of counter. + """ + raise NotImplementedError('_computed_sample() must be implemented in a subclass.') + + +class _TotalCounter(_BaseCounter): + """Counter type which provides a running total for the duration of the program.""" + def __init__(self, name, description): + super(_TotalCounter, self).__init__(name, description) + self._counter = 0L + + def increment(self, value=1): + """Increments the internal counter by a value. If not value is provided, increments + by one. + """ + self._counter += value + + def decrement(self, value=1): + """Decrements the internal counter by a value. If not value is provided, decrements + by one. + """ + self._counter -= value + + def get_total(self): + return self._counter + + def _raw_sample(self): + # Raw sample is simply the current value of the counter. + return self._counter + + def _computed_sample(self, s1, s2): + # For total, do not consider the earlier value - just return the current value. + return s2 + + +class _DeltaCounter(_TotalCounter): + """Counter type which provides the accumulation since the previous sample of the counter.""" + + def _computed_sample(self, s1, s2): + # For delta, subtract the previous sample from the current sample. + return s2 - s1 + + +class _RateCounter(_TotalCounter): + """Counter type which provides the rate of accumulation since the previous sample of the counter. + The rate is expressed in terms of a unit of time provided in seconds; the default is one second. + """ + def __init__(self, name, description, unit_seconds=1, time_func=None): + super(_RateCounter, self).__init__(name, description) + self._resolution = unit_seconds + self._time_func = time_func or time.time + + def _raw_sample(self): + # Raw sample for Rate must include both the current clock and the counter value. + return (self._counter, self._time_func()) + + def _computed_sample(self, s1, s2): + # Take the delta for the counter value and divide it by the elapsed time since the previous sample. + # The result is multiplied by the resolution value in order to provide the correct units. + time_diff = s2[1] - s1[1] + if time_diff == 0: + return 0 + return (s2[0] - s1[0]) * self._resolution / time_diff + + +class _AverageCounter(_BaseCounter): + """Counter type which provides the average value of some quantity over a number of occurences.""" + def __init__(self, name, description): + super(_AverageCounter, self).__init__(name, description) + self._counter = 0L + self._base_counter = 0L + + def add(self, value): + """Adds the value from a single occurrence to the counter.""" + self._counter += value + self._base_counter += 1 + + def _raw_sample(self): + return (self._counter, self._base_counter) + + def _computed_sample(self, s1, s2): + base_diff = s2[1] - s1[1] + if base_diff == 0: + return 0 + return (s2[0] - s1[0]) / base_diff + + +class Meter(object): + """Meter object, used to periodically sample all performance counters in a given namespace. + Once created, samples can be obtained by periodically calling the sample() method of the + Meter object. + + # Example: + import counters + + r = counters.define_rate('module.rate', 'Rate counter') + d = counters.define_delta('module.delta', 'Delta counter') + + m = counters.Meter() + m.add_counters(counters.counters.module) + + sample = m.sample() + print sample.module.rate # 0 + print sample.module.delta # 0 + + desc = m.describe() + + print sample.module.rate # 'Rate counter' + print sample.module.delta # 'Delta counter' + """ + def __init__(self, counters=None): + """Initialize a new meter object. If the optional counters parameter is provided, + its value is passed immediately to the add_counters() method. + """ + self._counters = dict() + self._description = None + if counters is not None: + self.add_counters(counters) + + def add_counters(self, counters): + """Add an additional counter or collection of counters to this meter. The intention is + for a portion of the global 'counters' instance (or another CounterManager object) to be + passed to this method. + """ + if isinstance(counters, DotDict): + flat = counters.flatten() + self._counters.update([(v, v.get_sampler()) for v in flat.itervalues()]) + else: + # Single counter instance. + self._counters[counters] = counters.get_sampler() + + # Clear existing description object. + self._description = None + + def sample(self): + """Samples all counters being tracked by this meter, returning a DotDict object + with all of the sampled values organized by namespace. + """ + new_sample = DotDict() + for k in self._counters.keys(): + new_sample[k.name] = self._counters[k]() + return new_sample + + def describe(self): + """Returns the description of all counters being tracked by this meter. The returned + object is a DotDict object with all of the descriptions organized by counter namespace. + """ + if self._description is None: + new_description = DotDict() + for k in self._counters.keys(): + new_description[k.name] = k.description + self._description = new_description + return self._description diff --git a/backend/base/daemon.py b/backend/base/daemon.py new file mode 100644 index 0000000..c5d873e --- /dev/null +++ b/backend/base/daemon.py @@ -0,0 +1,139 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" +Utility class which helps run a process as a daemon. + +Designed to be used as part of the Viewfinder asynchronous initialization +process. +""" +from __future__ import absolute_import + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import errno +import io +import signal +import os +import sys +import daemon + +from lockfile import pidlockfile +from functools import partial +from tornado import options + +options.define('daemon', 'none', + help='Specifies an option for running a daemon process. Should be one of: ' + '(start, stop, restart)') + + +class DaemonError(Exception): + """Represents errors from the daemon manager. Should only be raised during the process + of stopping or starting a daemon process, and should not be raised once then actual + daemon process is running.""" + pass + + +class DaemonManager(object): + """Class which manages the running of a process as a daemon. + + Depending on the value of the command-line value of the --daemon option, this class + can optionally run a process as a daemon, stop a running daemon process, or restart + it. The uniqueness of a daemon is enforced through use of a named lockfile - operations + which check for a currently running daemon will do so using this lockfile. + + When the 'start' option is specified, the current process will be converted to a daemon. + If an existing daemon process is already running, this process will fail with an + exception. + + When 'stop' is specified, any existing daemon process will be aborted and this process + will end. If no daemon was running, an error will be raised. + + When 'restart' is specified, any existing daemon process will be stopped and the current + process will be converted to a daemon. + """ + action_funcs = ('start', 'stop', 'restart') + + def __init__(self, lock_file_name): + self.lockfile = pidlockfile.PIDLockFile(lock_file_name) + + def SetupFromCommandLine(self, run_callback, shutdown_callback): + """Configures the daemon based on the command line --daemon option. + + If no option is specified, then run_callback is invoked with shutdown_callback + as a parameter. + + If 'start' or 'restart' is specified, then the current process will be converted + to a daemon before invoking run_callback. + + If 'stop' is specified, then any running daemon will be terminated and the + shutdown_callback will be invoked.""" + opt = options.options.daemon.lower() + + if opt == 'none': + # Not running as a daemon, run callback immediately. + run_callback(shutdown_callback) + return + + def _shutdown_daemon(): + self._context.close() + shutdown_callback() + + opt = options.options.daemon.lower() + if opt == 'start': + self.StartDaemon() + run_callback(_shutdown_daemon) + elif opt == 'stop': + self.StopDaemon(True) + shutdown_callback() + elif opt == 'restart': + self.StopDaemon(False) + self.StartDaemon() + run_callback(_shutdown_daemon) + else: + raise ValueError('"%s" is not a valid choice for the --daemon option. Must be one of %s' + % (opt, self.action_funcs)) + + def StartDaemon(self): + """Converts the current process to a daemon unless another daemon process + is already running on the system. + """ + current_pid = self._get_current_pid() + if current_pid is not None: + raise DaemonError('Daemon process is already started with PID:%s' % current_pid) + + self._context = daemon.daemon.DaemonContext(pidfile=self.lockfile) + try: + self._context.open() + except pidlockfile.AlreadyLocked: + pid = self.lockfile.read_pid() + raise DaemonError('Daemon process is already started with PID:%s' % pid) + + def StopDaemon(self, require_running=True): + """Stops any currently running daemon process on the system.""" + current_pid = self._get_current_pid() + if require_running and current_pid is None: + raise DaemonError('Daemon process was not running.') + + try: + os.kill(current_pid, signal.SIGTERM) + except OSError, exc: + raise DaemonError('Failed to stop daemon process %d: %s' % (current_pid, exc)) + + def _get_current_pid(self): + """Get the process ID of any currently running daemon process. Returns + None if no process is running. + """ + current_pid = None + if self.lockfile.is_locked(): + current_pid = self.lockfile.read_pid() + if current_pid is not None: + try: + # A 0 signal will do nothing if the process exists. + os.kill(current_pid, 0) + except OSError, exc: + if exc.errno == errno.ESRCH: + # PID file was stale, delete it. + current_pid = None + self.lockfile.break_lock() + + return current_pid diff --git a/backend/base/debug.py b/backend/base/debug.py new file mode 100644 index 0000000..51f6218 --- /dev/null +++ b/backend/base/debug.py @@ -0,0 +1,61 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Debug utilities. + +http://www.smira.ru/wp-content/uploads/2011/08/heapy.html + + - HeapDebugger: class that runs periodic guppy heap analysis +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +from functools import partial +from tornado import ioloop, options + +options.define('profile_interval', default=None, help='seconds between heap profiles') + +_can_use_guppy = False +try: + from guppy import hpy + _can_use_guppy = True +except: + logging.warning('unable to import guppy module for heap analysis') + + +class HeapDebugger(object): + """If guppy can be imported, creates a periodic + """ + def __init__(self): + if options.options.profile_interval: + hp = self.StartProfiling() + interval_ms = int(options.options.profile_interval) * 1000 + self._periodic_cb = ioloop.PeriodicCallback( + partial(self._PeriodicDump, hp), interval_ms, ioloop.IOLoop.current()) + self._periodic_cb.start() + + def StartProfiling(self): + """Creates a new heapy object, sets it to begin profiling, and returns to caller. + """ + hp = hpy() + hp.setrelheap() + return hp + + def StopProfiling(self, hp): + """Returns the heap object for further examination.""" + try: + return hp.heap() + finally: + del hp + + def _PeriodicDump(self, hp): + """Called from a periodic timer to dump (hopefully) useful information + about the heap to the logs. + """ + logging.info('in periodic dump') + heap = self.StopProfiling(hp) + logging.info('By class or dict owner:\n%s' % heap.byclodo) + logging.info('By referrers:\n%s' % heap.byrcs) + logging.info('By type:\n%s' % heap.bytype) + logging.info('By via:\n%s' % heap.byvia) + del heap diff --git a/backend/base/dotdict.py b/backend/base/dotdict.py new file mode 100644 index 0000000..7e46c55 --- /dev/null +++ b/backend/base/dotdict.py @@ -0,0 +1,105 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" Defines DotDict data structure, a nestable dictionary structure which allows programmers +to access nested variables using '.' notation. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + + +class DotDict(dict): + """Dictionary extension which allows referencing nested dictionaries using object dot notation. + + Pilfered from: + http://stackoverflow.com/questions/3797957/python-easily-access-deeply-nested-dict-get-and-set + + Examples: + + d = DotDict({'a':{'b':1,'c':2}, 'd':3}) + print d.a.b # 1 + print d.a.c # 2 + print d.a.d # Raises KeyError + print d.d # 3 + d.a.e = 3 # Value set correctly + + # Limitation - cannot set sparsely using dot notation: + d.x.y.z = 3 # Fails with KeyError + d['x.y.z'] = 3 # Works correctly + print d.x.y.z # 3 + """ + def __init__(self, value=None): + if value is None: + pass + elif isinstance(value, dict): + for key in value: + self.__setitem__(key, value[key]) + else: + raise TypeError, 'Can only initialize dotdict from another dict.' + + def __setitem__(self, key, value): + if '.' in key: + myKey, restOfKey = key.split('.', 1) + target = self.setdefault(myKey, DotDict()) + if not isinstance(target, DotDict): + raise KeyError('Cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))) + target[restOfKey] = value + else: + if isinstance(value, dict) and not isinstance(value, DotDict): + value = DotDict(value) + dict.__setitem__(self, key, value) + + def __getitem__(self, key): + if '.' not in key: + return dict.__getitem__(self, key) + myKey, restOfKey = key.split('.', 1) + target = dict.__getitem__(self, myKey) + if not isinstance(target, DotDict): + raise KeyError('Cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))) + return target[restOfKey] + + def __contains__(self, key): + if '.' not in key: + return dict.__contains__(self, key) + myKey, restOfKey = key.split('.', 1) + if not dict.__contains__(self, myKey): + return False + target = dict.__getitem__(self, myKey) + if not isinstance(target, DotDict): + return False + return restOfKey in target + + def flatten(self): + """Returns a regular dictionary with the same leaf items as this DotDict. + The resulting dict will have no nesting - items will have a dot-joined key + reflecting their nesting in the DotDict. + + #Example... + d = DotDict() + d.a = 1 + d.x = dict() + d.x.y = 2 + d.x.z = 3 + + flat = d.flatten() + print d.keys() # ['a', 'x.y', 'x.z'] + """ + newdict = dict() + def recurse_flatten(prefix, dd): + for k, v in dd.iteritems(): + newkey = prefix + '.' + k if len(prefix) > 0 else k + if isinstance(v, DotDict): + recurse_flatten(newkey, v) + else: + newdict[newkey] = v + + recurse_flatten('', self) + return newdict + + + def setdefault(self, key, default): + if key not in self: + self[key] = default + return self[key] + + __setattr__ = __setitem__ + __getattr__ = __getitem__ diff --git a/backend/base/environ.py b/backend/base/environ.py new file mode 100755 index 0000000..31261c0 --- /dev/null +++ b/backend/base/environ.py @@ -0,0 +1,228 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" Module to collect details about current environment and make these details available to other modules. + +Initially, the main purpose of this is to distinguish a server's role as far as production versus staging. The first +use of this information (along with user data) is to determine if a request should be redirected to the production +or staging cluster. + +If running on a dev machine, the '--devbox' option should be provided so that failure to contact AWS/EC2 for +metadata and EC2 tags won't cause a startup failure. +""" + +__author__ = 'mike@emailscrubbed.com (Mike Purtell)' + +import atexit +import boto +import logging +import shutil +import sys +import tempfile +import time +import os + +from viewfinder.backend.base import base_options +from viewfinder.backend.base.ami_metadata import GetAMIMetadata, SetAMIMetadata, Metadata +from viewfinder.backend.base.exceptions import * +from tornado import options, ioloop +from secrets import GetSecret, InitSecrets +from viewfinder.backend.base import base_options # imported for option definitions + +# Holds global ServerEnvironment object +_server_environment = None + +# Retry interval while trying to retrieve EC2 instance tags. This only happens during server startup and +# 10 seconds allows for transient AWS infrastructure problems to clear up without slamming the infrastructure. +_EC2_TAGS_RETRIEVAL_RETRY_SECS = 10 + +# These values are stored in the EC2 tag, NodeType. +_EC2_TAG_NODETYPE_PRODUCTION = 'PROD' +_EC2_TAG_NODETYPE_STAGING = 'STAGING' + +# This directory should be availalbe on all EC2 instances. It's a 100GB EBS volume. +_EC2_TEMP_DIR = '/mnt/vf_temp' + +class ServerEnvironment(object): + """Encapsulates collection of server environment details.""" + def __init__(self, is_devbox, is_staging): + self._is_devbox = is_devbox + self._is_staging = is_staging + self._staging_host = 'staging.%s' % options.options.domain + self._prod_host = '%s.%s' % (options.options.www_label, options.options.domain) + self._vf_temp_dir = options.options.vf_temp_dir if is_devbox else _EC2_TEMP_DIR + + @staticmethod + def IsDevBox(): + """Returns true if running as if on a developer machine.""" + assert _server_environment is not None + return _server_environment._is_devbox + + @staticmethod + def IsStaging(): + """Returns true if running as a staging server.""" + assert _server_environment is not None + return _server_environment._is_staging + @staticmethod + def GetHost(): + """Gets name of current host. This will be staging. if running as staging server, + or www. if running as production server. + """ + assert _server_environment is not None + return _server_environment._staging_host if _server_environment._is_staging else _server_environment._prod_host + + @staticmethod + def GetViewfinderTempDirPath(): + """Gets the path to the temp dir that viewfinder should use as temp. + """ + assert _server_environment is not None + if not _server_environment._vf_temp_dir: + # If running as production, this should already be set. + # Even if this is devbox, it may have been set explicitly with the vf_temp_dir option. + assert _server_environment._is_devbox + # This is commonly the path that will be taken for tests. + _server_environment._vf_temp_dir = tempfile.mkdtemp() + atexit.register(shutil.rmtree, _server_environment._vf_temp_dir) + + return _server_environment._vf_temp_dir + + @staticmethod + def GetRedirectHost(): + """Gets name of host to which staging/production users are redirected if they don't "match" + the current host. This will be staging. if running as production server, or + www. if running as staging server. + """ + assert _server_environment is not None + return _server_environment._prod_host if _server_environment._is_staging else _server_environment._staging_host + + @staticmethod + def InitServerEnvironment(): + """Collects information during startup about the current server environment. + This will retry on transient errors and assert for non-transient issues such as mis-configuration. + """ + global _server_environment + + # Start with options settings. + is_devbox = options.options.devbox + is_staging = options.options.is_staging + + # Determine whether running as a staging server if not specified on the command-line. + if is_staging is None: + if is_devbox: + # This is used to skip trying to retrieve AWS metadata and primarily for desktop development. + is_staging = False + else: + # No override has been provided, so dynamically determine which type of instance we're on. + + # Get the AWS/EC2 instance id for the instance this is executing on. + instance_id = GetAMIMetadata().get('meta-data/instance-id', None) + if instance_id is None: + raise ViewfinderConfigurationError( + "We should have already retrieved the instance id by this point. " + + "If running on dev box, use the --devbox option.") + + reservations = None + + # Connect to EC2 and get instance tags. This is done synchronously as there's nothing else the server + # should be doing until this has completed successfully. + while not reservations: + try: + logging.info("Connecting to EC2 to retrieve instance tags") + ec2conn = boto.connect_ec2(aws_access_key_id=GetSecret('aws_access_key_id'), + aws_secret_access_key=GetSecret('aws_secret_access_key')) + logging.info("Querying EC2 for instance data for instance: %s" % instance_id) + reservations = ec2conn.get_all_instances([instance_id, ]) + except Exception as e: + logging.warning("Exception while trying to retrieve EC2 instance tags: %s: %s, %s", + type(e), e.message, e.args) + time.sleep(_EC2_TAGS_RETRIEVAL_RETRY_SECS) + else: + if reservations is None: + logging.warning("Empty result while trying to retrieve EC2 instance tags.") + time.sleep(_EC2_TAGS_RETRIEVAL_RETRY_SECS) + + if len(reservations) != 1: + raise ViewfinderConfigurationError("Should have gotten one and only one reservation.") + reservation = reservations[0] + if len(reservation.instances) != 1: + raise ViewfinderConfigurationError("There should be one and only one instance in this reservation.") + instance = reservation.instances[0] + if instance is None: + raise ViewfinderConfigurationError("Instance not found in reservation metadata.") + if instance.__dict__.get('id', None) != instance_id: + raise ViewfinderConfigurationError( + "instance id in reservation metadata doesn't match expected value: %s vs. %s." % + (instance.__dict__.get('id', None), instance_id)) + + # Enumerate all the EC2 tags and their values to the log. + for tagName in instance.__dict__['tags']: + logging.info("This EC2 Instance Tag[%s]: '%s'" % (tagName, instance.__dict__['tags'][tagName])) + + nodetype = instance.__dict__['tags'].get('NodeType', None) + + logging.info("Retrieved instance tag for NodeType: %s" % nodetype) + + if nodetype == _EC2_TAG_NODETYPE_PRODUCTION: + is_staging = False + elif nodetype == _EC2_TAG_NODETYPE_STAGING: + is_staging = True + else: + raise ViewfinderConfigurationError("Invalid EC2 tag for NodeType on this instance. Tag: %s" % nodetype) + + if is_devbox: + logging.info("server starting as DevBox instance.") + + if is_staging: + _server_environment = ServerEnvironment(is_devbox, is_staging) + logging.info('server starting as Staging instance: %s' % _server_environment.GetHost()) + else: + _server_environment = ServerEnvironment(is_devbox, is_staging) + logging.info('server starting as Production instance: %s' % _server_environment.GetHost()) + + # Make a record in the log of what revision was last copied to this instance. + hg_revision = ServerEnvironment.GetHGRevision() + logging.info("Hg revision: %s" % hg_revision) + + @staticmethod + def GetHGRevision(): + """Attempts to retrieve the current mercurial revision number from the local + filesystem. + """ + filename = os.path.join(os.path.dirname(__file__), '../../hg_revision.txt') + try: + with open(filename) as f: + return f.read().strip() + except IOError: + return None + + +def main(): + """Test/Exercise ServerEnvironment on EC2 instance. + Initializes AMI Metadata and initializes ServerEnvironment object followed by output of results. + """ + query_ip = Metadata._QUERY_IP + + options.parse_command_line() + + def _OnInitSecrets(): + ServerEnvironment.InitServerEnvironment() + + if ServerEnvironment.IsDevBox(): + print "IsDevBox environment" + if ServerEnvironment.IsStaging(): + print "IsStaging environment" + else: + print "IsProduction environment" + + ioloop.IOLoop.current().stop() + + def _MetadataCallback(metadata): + SetAMIMetadata(metadata) + InitSecrets(_OnInitSecrets, False) + + Metadata(callback=_MetadataCallback, query_ip=query_ip) + ioloop.IOLoop.current().start() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/base/exceptions.py b/backend/base/exceptions.py new file mode 100644 index 0000000..31aeb9b --- /dev/null +++ b/backend/base/exceptions.py @@ -0,0 +1,145 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder exceptions. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from viewfinder.backend.resources.message.error_messages import ErrorDef + + +class ViewfinderError(Exception): + """Base class for viewfinder exceptions.""" + def __init__(self, error_def, **error_args): + if isinstance(error_def, ErrorDef): + self.id = error_def.id + message = error_def.format % error_args + else: + assert isinstance(error_def, basestring) + assert not error_args + self.id = None + message = error_def + + super(ViewfinderError, self).__init__(message) + +class AdminAPIError(ViewfinderError): + """Administration API failure.""" + pass + +class IdentityUnreachableError(ViewfinderError): + """Identity could not be contacted for invitation.""" + pass + +class EmailError(ViewfinderError): + """Failure to send email as specified.""" + pass + +class SMSError(ViewfinderError): + """Failure to send SMS as specified.""" + pass + +class MigrationError(ViewfinderError): + """Unable to upgrade database item.""" + pass + +class HttpForbiddenError(ViewfinderError): + """Errors derived from this will result in a 403 error being returned to the client. + """ + +class PermissionError(HttpForbiddenError): + """Permissions do not exist for intended action.""" + pass + +class TooManyGuessesError(HttpForbiddenError): + """Too many incorrect attempts have been made to guess a password or other secret.""" + pass + +class ExpiredError(HttpForbiddenError): + """The requested resource is no longer available because it has expired.""" + pass + +class TooManyRetriesError(ViewfinderError): + """An operation has retried too many times and is being aborted.""" + pass + +class CannotReadEncryptedSecretError(ViewfinderError): + """The secrets in the secrets directory require a passphrase for + decryption. + """ + pass + +class TooManyOutstandingOpsError(ViewfinderError): + """Too many operations are outstanding. This prevents the + server running out of memory by enforcing flow control to + requesting clients. + """ + pass + +class DBProvisioningExceededError(ViewfinderError): + """The database limits on capacity units for a table were exceeded. + The client should backoff and retry. + """ + pass + +class DBLimitExceededError(ViewfinderError): + """The database limits on capacity units for a table were exceeded. + The client should backoff and retry. + """ + pass + +class DBConditionalCheckFailedError(ViewfinderError): + """The database cannot complete the request because a conditional + check attached to the request failed. + """ + pass + +class InvalidRequestError(ViewfinderError): + """The request contains disallowed or malformed fields or values. This + error indicates a buggy or potentially malicious client. + """ + pass + +class NotFoundError(ViewfinderError): + """The request references resources that cannot be found. This error indicates a buggy or + malicious client. + """ + pass + +class CannotWaitError(ViewfinderError): + """Cannot wait for the operation to complete, because another server + is already running an operation for this user. + """ + pass + +class LockFailedError(ViewfinderError): + """Cannot acquire lock because it has already been acquired by another + agent. + """ + pass + +class FailpointError(ViewfinderError): + """Operation failed due to a deliberately triggered failure (for testing purposes).""" + def __init__(self, filename, lineno): + super(FailpointError, self).__init__('Operation failpoint triggered.') + self.filename = filename + self.lineno = lineno + +class ViewfinderConfigurationError(ViewfinderError): + """There is something wrong with either server options and/or environmental + configuration, such as viewfinder configuration stored in AWS metadata. + """ + pass + +class LimitExceededError(HttpForbiddenError): + """Client request attempted some action which would have exceeded a limit. + """ + pass + +class ServiceUnavailableError(ViewfinderError): + """The service is temporarily unavailable.""" + pass + +class StopOperationError(ViewfinderError): + """Stop the current operation in order to run a nested operation.""" + def __init__(self): + super(StopOperationError, self).__init__('Current operation stopped.') diff --git a/backend/base/handler.py b/backend/base/handler.py new file mode 100755 index 0000000..e5aa964 --- /dev/null +++ b/backend/base/handler.py @@ -0,0 +1,97 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Provides decorators for request handler methods: + + @handler.asynchronous: similar to tornado.web.asynchronous. It disables the auto-completion + on handler exit, but also optionally provides a client stub to the + DynamoDB datastore backend, and to the photo object store. + + @handler.authenticated: similar to tornado.web.authenticated, except that it raises HTTP + 401 rather than 403, and allows additional options. + +Example usage: + +class MyApp(tornado.web.RequestHandler): + @handler.authenticated() + @handler.asynchronous(datastore=True) + def get(self): + self.write(self._client(...)) + self.finish() +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import functools +import logging +import urllib +import urlparse + +from tornado import web +from viewfinder.backend.db.db_client import DBClient +from viewfinder.backend.db.user import User +from viewfinder.backend.storage.object_store import ObjectStore +from viewfinder.backend.base import util + + +def asynchronous(datastore=False, obj_store=False, log_store=False): + """Wrap request handler methods with this decorator if they will require asynchronous + access to DynamoDB datastore or S3 object store for photo storage. + + If datastore=True, then a DynamoDB client is available to the handler as self._client. If + obj_store=True, then an S3 client for the photo storage bucket is available as self._obj_store. + If log_store is true, then an S3 client for the user log storage bucket is available as + self._log_store + + Like tornado.web.asynchronous, this decorator disables the auto-finish functionality. + """ + def _asynchronous(method): + def _wrapper(self, *args, **kwargs): + """Disables automatic HTTP response completion on exit.""" + self._auto_finish = False + if datastore: + self._client = DBClient.Instance() + if obj_store: + self._obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO) + if log_store: + self._log_store = ObjectStore.GetInstance(ObjectStore.USER_LOG) + + with util.ExceptionBarrier(self._stack_context_handle_exception): + return method(self, *args, **kwargs) + + return functools.wraps(method)(_wrapper) + return _asynchronous + + +def authenticated(allow_prospective=False): + """Wrap request handler methods with this decorator to require that the user be logged in. + Raises an HTTP 401 error if not. + + This method is exactly the same as tornado.web.authenticated, except that 401 is raised + instead of 403. This is important, because the clients will re-authenticate only if they + receive a 401. + + If allow_prospective=False, then prospective user cookies are not authorized access. + WARNING: Before changing allow_prospective to True, make certain to think through the + permissions that a prospective user should have for that particular handler. + """ + def _authenticated(method): + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if not self.current_user: + if self.request.method in ("GET", "HEAD"): + url = self.get_login_url() + if "?" not in url: + if urlparse.urlsplit(url).scheme: + # if login url is absolute, make next absolute too + next_url = self.request.full_url() + else: + next_url = self.request.uri + url += "?" + urllib.urlencode(dict(next=next_url)) + self.redirect(url) + return + raise web.HTTPError(401, 'You are not logged in. Only users that have logged in can access this page.') + elif isinstance(self.current_user, User) and not allow_prospective and not self.current_user.IsRegistered(): + raise web.HTTPError(403, 'You are not a registered user. Sign up for Viewfinder to gain access to this page.') + return method(self, *args, **kwargs) + return wrapper + return _authenticated diff --git a/backend/base/keyczar_dict.py b/backend/base/keyczar_dict.py new file mode 100644 index 0000000..01ef933 --- /dev/null +++ b/backend/base/keyczar_dict.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Provides an implementation of a Keyczar keyset reader and writer +that stores meta information and versioned keys in a Python dict. The +dict can then be stored in our database or in files in the secrets +directory. The attributes of the dict are as follows: + meta - contains the keyset metadata as a string + 1 - contains the first key in the set (if it exists) + 2 - contains the second key in the set (if it exists) + ...and so on +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import json + +from keyczar import errors, readers, writers + + +class DictReader(readers.Reader): + """Keyczar reader that reads key data from a Python dict.""" + def __init__(self, keydata): + """Construct reader from either a JSON string or a Python dict.""" + if isinstance(keydata, basestring): + keydata = json.loads(keydata) + assert isinstance(keydata, dict), keydata + self.dict = keydata + + def GetMetadata(self): + """Returns the "meta" attribute.""" + return self.dict['meta'] + + def GetKey(self, version_number): + """Returns a key having "version_number" as its name.""" + return self.dict[str(version_number)] + + def Close(self): + """Does nothing, as there is nothing to close.""" + pass + + +class DictWriter(writers.Writer): + """Keyczar writer that writes key data to a Python dict.""" + def __init__(self, keydata=None): + """Construct reader from either a JSON string or a Python dict.""" + if isinstance(keydata, basestring): + keydata = json.loads(keydata) + assert keydata is None or isinstance(keydata, dict), keydata + self.dict = keydata if keydata is not None else {} + + def WriteMetadata(self, metadata, overwrite=True): + """Stores "metadata" in the "meta" attribute.""" + if not overwrite and 'meta' in metadata: + raise errors.KeyczarError('"meta" attribute already exists') + self.dict['meta'] = str(metadata) + + def WriteKey(self, key, version_number, encrypter=None): + """Stores "key" in an attribute having "version_number" as its name.""" + key = str(key) + if encrypter: + key = encrypter.Encrypt(key) # encrypt key info before outputting + self.dict[str(version_number)] = key + + def Remove(self, version_number): + """Removes the key for the given version.""" + self.dict.pop(str(version_number)) + + def Close(self): + """Does nothing, as there is nothing to close.""" + pass diff --git a/backend/base/logging_utils.py b/backend/base/logging_utils.py new file mode 100644 index 0000000..bb00981 --- /dev/null +++ b/backend/base/logging_utils.py @@ -0,0 +1,66 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Common logging utilities and code. + + - FORMATTER: default Viewfinder log formatter + - ErrorLogFilter: Filter used to count errors from a central location. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'matt@emailscrubbed.com (Matt Tracy)'] + +import datetime +import logging +import counters +import sys + + +class _LogFormatter(logging.Formatter): + converter = datetime.datetime.fromtimestamp + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + s = ct.strftime(datefmt) + else: + t = ct.strftime('%Y-%m-%d %H:%M:%S') + s = '%s:%03d' % (t, record.msecs) + return s + + def format(self, record): + """Ensure that the log is consistently encoded as UTF-8.""" + msg = super(_LogFormatter, self).format(record) + if isinstance(msg, unicode): + msg = msg.encode('utf-8') + return msg + +FORMATTER = _LogFormatter('%(asctime)s [pid:%(process)d] %(module)s:%(lineno)d: %(message)s') + + +class StdStreamProxy(object): + """Proxy for sys.std{out,err} to ensure it can be updated. + + The logging module (at least in its default configuration) makes + a copy of sys.std{out,err} at startup and writes to those objects. + This is incompatible with unittest's "buffer" feature, which points + the variables in sys to new values. By wrapping sys.std{out,err} + with this proxy before logging is configured, we ensure that + unittest's changes have the desired effect. + + Usage: Before logging is initialized (i.e. before + tornado.options.parse_command_line), do + sys.stdout = StdStreamProxy('stdout') + sys.stderr = StdStreamProxy('stderr') + """ + def __init__(self, name): + assert name in ('stdout', 'stderr') + self.name = name + self.real_stream = getattr(sys, name) + + def __getattr__(self, name): + current_stream = getattr(sys, self.name) + if current_stream is self: + # not redirected, so write through to the original stream + return getattr(self.real_stream, name) + else: + # write to the new current stream + return getattr(current_stream, name) diff --git a/backend/base/main.py b/backend/base/main.py new file mode 100644 index 0000000..3d0c0d3 --- /dev/null +++ b/backend/base/main.py @@ -0,0 +1,140 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utilities for base class of servers and command-line tools. + + - InitAndRun() - initializes secrets, invokes a run callback. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import os +import signal +import sys + +from functools import partial +from tornado import gen, httpclient, options, stack_context, ioloop +from viewfinder.backend.base import ami_metadata, process_util, secrets +from viewfinder.backend.base.environ import ServerEnvironment +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.storage import object_store, server_log + +options.define('blocking_log_threshold', default=None, type=float, + help='Log a warning (with stack trace) if the IOLoop is blocked for this many seconds') + + +@gen.coroutine +def _Init(init_db=True, server_logging=True): + """Completes Viewfinder initialization, such as secrets, DB client, AMI metadata, and the + object store. + """ + # Configure the default http client to use pycurl. + httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=100) + + # Retrieve AMI metadata before initializing secrets. Don't try to get metadata on devbox. + if options.options.devbox: + metadata = ami_metadata.Metadata() + else: + metadata = yield gen.Task(ami_metadata.Metadata) + + if metadata is None: + raise Exception('failed to fetch AWS instance metadata; if running on dev box, ' + 'use the --devbox option') + + ami_metadata.SetAMIMetadata(metadata) + logging.info('AMI metadata initialized') + + # Initialize server environment. + ServerEnvironment.InitServerEnvironment() + logging.info('server environment initialized') + + # Initialize secrets. + yield gen.Task(secrets.InitSecrets, can_prompt=sys.stderr.isatty()) + logging.info('secrets initialized') + + # Initialize database. + if init_db: + yield gen.Task(db_client.InitDB, vf_schema.SCHEMA) + logging.info('DB client initialized') + + # Initialize object store. + object_store.InitObjectStore(temporary=False) + logging.info('object store initialized') + + # Initialize the server log now that the object store is initialized. + if server_logging: + server_log.InitServerLog() + + logging.info('main.py initialization complete') + + +def InitAndRun(run_callback, shutdown_callback=None, init_db=True, server_logging=True): + """Initializes and configures the process and then the Viewfinder server. + + If 'init_db' is False, skip initializing the database client. If 'server_logging' is False, + do not write logs to S3 and do not override the logging level to INFO. + + Creates an instance of IOLoop, and adds it to the stack context and starts it. When + initialization is complete, invokes 'run_callback'. When that has completed, invokes + "shutdown_callback". + + Note that this function runs *synchronously*, and will not return until both "run_callback" + and "shutdown_callback" have been executed. + """ + options.parse_command_line() + + # Set the global process name from command line. Do not override if already set. + proc_name = os.path.basename(sys.argv[0]) + if proc_name and process_util.GetProcessName() is None: + process_util.SetProcessName(proc_name) + + # Create IOLoop and add it to the context. + # Use IOLoop.Instance() to avoid problems with certain third-party libraries. + io_loop = ioloop.IOLoop.instance() + if options.options.blocking_log_threshold is not None: + io_loop.set_blocking_log_threshold(options.options.blocking_log_threshold) + + # Setup signal handlers to initiate shutdown and stop the IOLoop. + def _OnSignal(signum, frame): + logging.info('process stopped with signal %d' % signum) + io_loop.stop() + + signal.signal(signal.SIGHUP, _OnSignal) + signal.signal(signal.SIGINT, _OnSignal) + signal.signal(signal.SIGQUIT, _OnSignal) + signal.signal(signal.SIGTERM, _OnSignal) + + @gen.coroutine + def _InvokeCallback(wrapped_callback): + """Wraps "run_callback" in function that returns the Future that IOLoop.run_sync requires.""" + yield gen.Task(wrapped_callback) + + # If this is true at shutdown time, exit with error code. + shutdown_by_exception = False + + try: + # Initialize. + if server_logging: + logging.getLogger().setLevel(logging.INFO) + logging.getLogger().handlers[0].setLevel(logging.INFO) + + io_loop.run_sync(partial(_Init, init_db=init_db, server_logging=server_logging)) + + # Run. + io_loop.run_sync(partial(_InvokeCallback, run_callback)) + + # Shutdown. + if shutdown_callback is not None: + io_loop.run_sync(partial(_InvokeCallback, shutdown_callback)) + except Exception as ex: + # TimeoutError is raised by run_sync if signal handler stopped the ioloop. + if not isinstance(ex, ioloop.TimeoutError): + logging.exception('unhandled exception in %s' % sys.argv[0]) + shutdown_by_exception = True + + db_client.ShutdownDB() + io_loop.run_sync(server_log.FinishServerLog) + + # Exit with error code if exception caused shutdown. + if shutdown_by_exception: + sys.exit(1) diff --git a/backend/base/message.py b/backend/base/message.py new file mode 100644 index 0000000..9301016 --- /dev/null +++ b/backend/base/message.py @@ -0,0 +1,1020 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Contains functions for validating and versioning messages. + +Messages (described below) need to be validated in order to ensure +they conform to a particular JSON schema. Also, this module provides +support for allowing and generating multiple versions of the same +message. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)' + 'andy@emailscrubbed.com (Andy Kimball)'] + +import re +import sys +import time +import validictory + +from functools import partial + +# The client uses the Unicode separator char class, but the Python re module does not support +# that, so just approximate. +FULL_NAME_RE = re.compile('\s*(\S+)\s*(.*)\s*', re.UNICODE) + + +class BadMessageException(Exception): + """Raised when an invalid message is encountered.""" + pass + + +class Message(object): + """A message is represented as a Python dictionary that is typically created + from JSON text and is destined to be serialized to a network or to a + database as JSON text. The dictionary may contain nested dictionaries or + arrays that form the message structure. Messages may be validated, + "sanitized", and migrated from a newer format to an older format (or vice- + versa). + """ + + INITIAL_VERSION = 0 + """Version of the starting message format. This format does not contain a + headers object or a version field in that object. Therefore, if a message + is encountered without the headers object, it is assumed to have this + version. + """ + + ADD_HEADERS_VERSION = 1 + """Add headers object to every message and version field.""" + + TEST_VERSION = 2 + """Version used for testing migrators.""" + + RENAME_EVENT_VERSION = 3 + """Rename fields from "event" to "episode".""" + + ADD_TO_VIEWPOINT_VERSION = 4 + """Episodes have new "viewpoint_id" and "publish_timestamp" fields that + older clients will not understand. + """ + + QUERY_EPISODES_VERSION = 5 + """QueryEpisodes now takes additional selection fields, and by default + photos are not selected. + """ + + UPDATE_POST_VERSION = 6 + """QueryEpisodes no longer returns a post_timestamp field.""" + + UPDATE_SHARE_VERSION = 7 + """The share operation adds support for viewpoints.""" + + ADD_OP_HEADER_VERSION = 8 + """Add op_id and op_timestamp to the headers of mutating operation + requests. + """ + + ADD_ACTIVITY_VERSION = 9 + """Add activity attribute to requests that require it.""" + + EXTRACT_MD5_HASHES = 10 + """Extract MD5 hashes from client_data to standalone attributes.""" + + INLINE_INVALIDATIONS = 11 + """Inline certain invalidations in notification messages.""" + + EXTRACT_FILE_SIZES = 12 + """Extract file sizes from client_data to standalone attributes.""" + + INLINE_COMMENTS = 13 + """Inline shorter comments in notifications.""" + + EXTRACT_ASSET_KEYS = 14 + """Extract asset keys from client_data to standalone attributes.""" + + SPLIT_NAMES = 15 + """Split full names into given and family name parts.""" + + EXPLICIT_SHARE_ORDER = 16 + """Client explicitly orders episodes and photos in share_new and share_existing.""" + + SUPPRESS_BLANK_COVER_PHOTO = 17 + """Remove cover_photo field if photo_id is blank (work around client bug).""" + + SUPPORT_MULTIPLE_IDENTITIES_PER_CONTACT = 18 + """Different results format for query_contacts related to support for upload_contacts.""" + + RENAME_PHOTO_LABEL = 19 + """Rename query_episodes photo label from HIDDEN back to REMOVED.""" + + SUPPRESS_AUTH_NAME = 20 + """Removes name fields from /link/viewfinder.""" + + SEND_EMAIL_TOKEN = 21 + """Include 4-digit access token in email rather than a button.""" + + SUPPORT_REMOVED_FOLLOWERS = 22 + """Return labels for each follower returned by query_viewpoints.""" + + SUPPRESS_COPY_TIMESTAMP = 23 + """Removes timestamp field from episodes in share and save operations.""" + + SUPPORT_CONTACT_LIMITS = 24 + """Truncates and skips various items during upload contacts in order to stay under limits.""" + + SUPPRESS_EMPTY_TITLE = 25 + """Removes empty title field from update_viewpoint request.""" + + # ----------------------------------------------------------------- + # Add new message versions here, making sure to update MAX_VERSION. + # Define an instance of the new migrator near the bottom of the + # file, and if it's a migrator that must always be run, then add + # it to REQUIRED_MIGRATORS. Any time you want to change messages, + # consider the following possible usages: + # 1. Messages to and from our service API. + # 2. Messages used by operation.py to persist operations. + # 3. Messages used by USER_UPDATES to store column names. + # 4. Messages used by notification.py for invalidations as + # well as notification arguments. + # ----------------------------------------------------------------- + + MAX_VERSION = SUPPRESS_EMPTY_TITLE + """This should always be set to the maximum message version.""" + + + _VERSION_HEADER_SCHEMA = { + 'description': 'defines message with optional headers object', + 'type': 'object', + 'properties': { + 'headers': { + 'description': 'defines headers object with required version field', + 'type': 'object', + 'properties': { + 'version': { + 'description': 'version of the message format', + 'type': 'integer', + 'minimum': ADD_HEADERS_VERSION, + }, + 'min_required_version': { + 'description': 'minimum required message format version that must be supported by the server', + 'type': 'integer', + 'required': False, + 'minimum': INITIAL_VERSION, + }, + }, + }, + }, + } + """Define validation schema so that header fields in the version object + are validated "enough" so that we can pick out the version without + worrying about getting exceptions that something doesn't exist, or is + the wrong data-type, etc. Other header fields are ignored at this stage; + it's just a private schema not meant to be used outside this class. + """ + + def __init__(self, message_dict, min_supported_version=INITIAL_VERSION, + max_supported_version=MAX_VERSION, default_version=INITIAL_VERSION): + """Construct a new message from the provided Python dictionary. Determine + the version of the message and store it in the version field. A number of + checks are made to make sure that the version is valid. The first + requirement is that the message version falls within the range (inclusive) + [min_supported_version, max_supported_version] that was passed to this + method. However, the version of the message itself can be somewhat + flexible. The message specifies a "version" field, but it can also specify + a "min_required_version" field. This gives the server the latitude to + pick a version in the range [min_required_version, version]. The server + will find the overlap between these two ranges and pick the largest + version value possible that still falls within both ranges. If the ranges + are disjoint, then a BadMessageException will be raised. + + If no version is present in the message, assume it is "default_version". + This allows messages to be easily constructed using literal dictionaries, + without needing to always insert a version header. + """ + assert type(message_dict) is dict, (type(message_dict), message_dict) + self.dict = message_dict + + assert min_supported_version >= MIN_SUPPORTED_MESSAGE_VERSION + assert max_supported_version <= MAX_MESSAGE_VERSION + + self.original_version = self._GetMessageVersion(max_supported_version, default_version) + self.version = self.original_version + + # Verify that the message's version should be accepted. + if self.version > max_supported_version: + raise BadMessageException('Version %d of this message is not supported by the server.' % self.version + + ' The server only supports this message up to version %d.' % + max_supported_version) + + if self.version < min_supported_version: + raise BadMessageException('Version %d of this message is not supported by the server.' % self.version + + ' The server only supports this message starting at version %d.' % + min_supported_version) + + + def Validate(self, schema, allow_extra_fields=False): + """Validate that the message conforms to the specified schema. + If the "allow_extra_fields" argument is False, then fail the + validation if the message contains any extra fields that are + not specified explicitly in the schema. If validation fails, + then raise a BadMessageException. If the validation succeeds, + associate the schema with this message by saving it in the + "self.schema" field. + """ + assert schema, "A schema must be provided in order to validate." + try: + validictory.validate(self.dict, schema) + if not allow_extra_fields: + self._FindExtraFields(self.dict, schema, True) + self.schema = schema + except Exception as e: + raise BadMessageException(e.message), None, sys.exc_info()[2] + + def Sanitize(self): + """Remove any fields from the message that are not explicitly + allowed by the schema. This is used to remove extraneous fields + from objects which may have been added during message processing. + """ + assert self.schema, "No schema available. Sanitize may only be called after Validate has been called." + self._FindExtraFields(self.dict, self.schema, False) + + def Migrate(self, client, migrate_version, callback, migrators=None): + """Migrate this message's content to the format with version + "migrate_version". To do this, apply the "migrators" list in + sequence. Each migrator will mutate the content of the message to + conform to the next (or previous) version of the message format. + If migrators == None, the REQUIRED_MIGRATORS will be used by + default. The migrators list should already be merged with + REQUIRED_MIGRATORS and sorted. Example: + + migrators = sorted(REQUIRED_MIGRATORS + [MyMigrator(), MyOtherMigrator()]) + + When the migration is completed, "callback" is invoked with the + message as its only parameter. + """ + def _OnMigrate(intermediate_version): + """Called each time a migrator has been applied; keep invoking Migrate + until the final migrate version is reached. + """ + self.version = intermediate_version + + # Update the version header to be the target version. + if intermediate_version >= Message.ADD_HEADERS_VERSION: + self.dict['headers']['version'] = intermediate_version + + # Continue migrating. + self.Migrate(client, migrate_version, callback, migrators) + + assert migrate_version >= MIN_SUPPORTED_MESSAGE_VERSION + assert migrate_version <= MAX_MESSAGE_VERSION + + if migrators is None: + migrators = REQUIRED_MIGRATORS + assert len(migrators) > 0 and type(migrators[0]) is AddHeadersMigrator, \ + 'The first migrator is not AddHeadersMigrator. Did you forget to merge with REQUIRED_MIGRATORS and sort?' + + # If current message version is the same as the desired version, nothing more to do. + if self.version == migrate_version: + callback(self) + return + + # Migrate the message version to the target version. + migrator_count = len(migrators) + if self.version < migrate_version: + for i in xrange(migrator_count): + migrator = migrators[i] + assert i == 0 or migrators[i - 1].migrate_version < migrator.migrate_version + + # Break if we reach a migrator that is above the desired version. + if migrator.migrate_version > migrate_version: + break + + if self.version < migrator.migrate_version: + migrator.MigrateForward(client, self, partial(_OnMigrate, migrator.migrate_version)) + return + else: + for i in xrange(migrator_count, 0, -1): + migrator = migrators[i - 1] + assert i == migrator_count or migrators[i].migrate_version > migrator.migrate_version + + # Break if we reach migrators that are below the desired version. + if migrator.migrate_version <= migrate_version: + break + + if self.version >= migrator.migrate_version: + migrator.MigrateBackward(client, self, partial(_OnMigrate, migrator.migrate_version - 1)) + return + + # No migrators apply, so skip directly to target version. + _OnMigrate(migrate_version) + + def Visit(self, visitor): + """Recursively visit the fields of the message in a depth-first + order. Invoke the visitor for each field, passing the name of + the field and its value. If the handler returns None, then no + changes are made to the message. If the handler returns an + empty tuple (), then the field is removed from the message. If + the handler returns a (name, value) tuple, then the field is + replaced with the new name and value. + """ + self._VisitHelper(self.dict, visitor) + + def _VisitHelper(self, node, handler): + """Helper visitor that traverses the message tree.""" + if isinstance(node, dict): + for key, value in node.items(): + # Recursively visit the dictionary contents. + self._VisitHelper(value, handler) + + # Give handler a chance to modify the (key, value) pair. + result = handler(key, value) + if result is None: + # None, so do nothing to this field. + continue + + # Remove the field + del node[key] + + # Add new field if one was returned. + if len(result) == 2: + node[result[0]] = result[1] + elif isinstance(node, list): + # Recursively visit list contents. + for item in node: + self._VisitHelper(item, handler) + + def _FindExtraFields(self, message_dict, schema, raise_error): + """Recursively traverses the message, looking for extra fields that + are not explicitly allowed in the schema. If "raise_error" is True, + then raise a BadMessageException if such fields are found. Otherwise, + remove the fields from the message entirely. + """ + if schema['type'] == 'object': + assert isinstance(message_dict, dict) + for k in message_dict.keys(): + if 'properties' in schema: + if k not in schema['properties']: + if raise_error: + raise BadMessageException('Message contains field "%s", which is not present in the schema.' % k) + else: + del message_dict[k] + continue + if schema['properties'][k]['type'] in ('object', 'array'): + self._FindExtraFields(message_dict[k], schema['properties'][k], raise_error) + elif schema['type'] == 'array': + assert isinstance(message_dict, list) + for val in message_dict: + self._FindExtraFields(val, schema['items'], raise_error) + + def _GetMessageVersion(self, max_supported_version, default_version): + """Extract the version from the message headers. Usually this is just + the value of the "version" field. However, if that version is not + supported by the server, the value of the "min_required_version" + field is consulted. If the value of this field is less than or equal + to the max supported version, then the server can "fall back" to + its max supported version. + """ + # If no version is present in the message, add it if allowed to do so. + headers = self.dict.get('headers', None) + if headers is None: + if default_version == Message.INITIAL_VERSION: + return Message.INITIAL_VERSION + self.dict['headers'] = dict(version=default_version) + elif not headers.has_key('version'): + headers['version'] = default_version + + # Validate version header. + try: + validictory.validate(self.dict, Message._VERSION_HEADER_SCHEMA) + except Exception as e: + raise BadMessageException(e.message) + + # Calculate version based on "version" and "min_required_version" fields. + version = int(self.dict['headers']['version']) + + if self.dict['headers'].has_key('min_required_version'): + min_required_version = int(self.dict['headers']['min_required_version']) + if min_required_version > version: + raise BadMessageException('The "min_required_version" value (%d) ' % min_required_version + + 'cannot be greater than the "version" value (%d).' % version) + + # If version is not supported, then use highest supported version that is still >= + # the min_required_version. + if version > max_supported_version: + version = max(min_required_version, max_supported_version) + + return version + + +class MessageMigrator(object): + """Migrates a message's content to conform to the next (or previous) + version of the message format. This is useful because from time to time, + the format of a particular message changes. The server must accept and + generate the new format. However, for reasons of backwards-compatibility, + older message formats must also be accepted and generated by the server. + Migrators enable support for multiple message formats by forming a + pipeline of transform functions which are successively applied in order + to migrate a message from an older format to a newer format, or vice- + versa. + + As a message format changes, it should always maintain enough information + so that it can be migrated backwards to the minimum supported message + format. However, in the backwards-migration case, it is permissible to + lose information that was present in later formats (but was not + supported by earlier formats). + """ + def __init__(self, migrate_version): + """Construct a migrator that will be activated as a message's version + is migrated to or from "migrate_version". + """ + self.migrate_version = migrate_version + + def MigrateForward(self, client, message, callback): + """Called in order to migrate a message to the "migrate_version" format, + from the previous format version. "callback" is invoked with no parameters + when the migration is complete. + """ + raise NotImplementedError() + + def MigrateBackward(self, client, message, callback): + """Called in order to migrate a message from the "migrate_version" + format, to the previous format version. "callback" is invoked with no + parameters when the migration is complete. + """ + raise NotImplementedError() + + def __cmp__(self, other): + """Migrators are compared to one another by "migrate_version", which + imposes a total ordering of migrators. + """ + assert isinstance(other, MessageMigrator) + return cmp(self.migrate_version, other.migrate_version) + + +class AddHeadersMigrator(MessageMigrator): + """Migrator that adds a headers object to the message. The headers object + contains a single required "version" field. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.ADD_HEADERS_VERSION) + + def MigrateForward(self, client, message, callback): + """Add the headers object to the message.""" + message.dict['headers'] = dict(version=Message.ADD_HEADERS_VERSION) + callback() + + def MigrateBackward(self, client, message, callback): + """Remove the headers object from the message.""" + del message.dict['headers'] + callback() + + +class RenameEventMigrator(MessageMigrator): + """Migrator that renames all "event" fields in the message to + corresponding "episode" fields. + """ + _EVENT_FIELDS = ['event', 'events', 'event_id', 'event_ids', 'parent_event_id', 'original_event_id', + 'device_event_id', 'event_limit', 'event_start_key', 'last_event_key'] + _EPISODE_TO_EVENT = {field.replace('event', 'episode') : field for field in _EVENT_FIELDS} + _EVENT_TO_EPISODE = {field : field.replace('event', 'episode') for field in _EVENT_FIELDS} + + def __init__(self): + MessageMigrator.__init__(self, Message.RENAME_EVENT_VERSION) + + def MigrateForward(self, client, message, callback): + """Visit all fields in the message and replace event fields with episode fields.""" + def _ReplaceEventWithEpisode(key, value): + if RenameEventMigrator._EPISODE_TO_EVENT.has_key(key): + raise BadMessageException('Episode fields should not appear in older messages.') + episode = RenameEventMigrator._EVENT_TO_EPISODE.get(key, None) + return (episode, value) if episode else None + + message.Visit(_ReplaceEventWithEpisode) + callback() + + def MigrateBackward(self, client, message, callback): + """Visit all fields in the message and replace episode fields with event fields.""" + def _ReplaceEpisodeWithEvent(key, value): + if RenameEventMigrator._EVENT_TO_EPISODE.has_key(key): + raise BadMessageException('Event fields should not appear in newer messages.') + event = RenameEventMigrator._EPISODE_TO_EVENT.get(key, None) + return (event, value) if event else None + + message.Visit(_ReplaceEpisodeWithEvent) + callback() + + +class AddToViewpointMigrator(MessageMigrator): + """Migrator that removes new "viewpoint_id" and "publish_timestamp" + Episode attributes from responses to older clients. These attributes + were added to episode as part of removing the Published table. Since + it's only necessary to remove these attributes, there's no need to + implement MigrateForward. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.ADD_TO_VIEWPOINT_VERSION) + + def MigrateBackward(self, client, message, callback): + if 'episodes' in message.dict: + for ep in message.dict['episodes']: + del ep['viewpoint_id'] + del ep['publish_timestamp'] + callback() + + +class QueryEpisodesMigrator(MessageMigrator): + """Migrator that always sets the QueryEpisodes "get_photos" field + to True, since older clients expect the photos to be projected by + default. Since it's only necessary to do this on the incoming + request, there's no need to implement MigrateBackward. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.QUERY_EPISODES_VERSION) + + def MigrateForward(self, client, message, callback): + if 'episodes' in message.dict: + for ep in message.dict['episodes']: + ep['get_photos'] = True + callback() + + +class UpdatePostMigrator(MessageMigrator): + """Migrator that adds back the "post_timestamp" field to the + QueryEpisodes response message (so no need to implement MigrateForward). + """ + def __init__(self): + MessageMigrator.__init__(self, Message.UPDATE_POST_VERSION) + + def MigrateBackward(self, client, message, callback): + if 'episodes' in message.dict: + for ep in message.dict['episodes']: + if 'photos' in ep: + for ph in ep['photos']: + ph['post_timestamp'] = ph['timestamp'] + callback() + + +class UpdateShareMigrator(MessageMigrator): + """Migrator that adds support for viewpoints to the "share" operation, + as well as sharing photos from multiple episodes. + + NOTE: This migrator is now a no-op, because the new viewpoint support + was removed from the "share" operation, and instead put into "share_new" + and "share_existing" operations. + """ + pass + + +class AddOpHeaderMigrator(MessageMigrator): + """Migrator that adds op_id and op_timestamp headers to mutating operation + requests. The op_id is generated using the system device allocator so that + it's guaranteed to be globally unique. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.ADD_OP_HEADER_VERSION) + + def MigrateForward(self, client, message, callback): + from viewfinder.backend.db.operation import Operation + + def _OnAllocateId(id): + message.dict['headers']['op_id'] = id + message.dict['headers']['op_timestamp'] = time.time() + callback() + + Operation.AllocateSystemOperationId(client, _OnAllocateId) + + +class AddActivityMigrator(MessageMigrator): + """Migrator that adds activity attribute to operation requests which need + to create an activity. The activity_id is derived from the op_timestamp + and op_id headers. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.ADD_ACTIVITY_VERSION) + + def MigrateForward(self, client, message, callback): + from viewfinder.backend.db.activity import Activity + from viewfinder.backend.db.operation import Operation + + timestamp = message.dict['headers']['op_timestamp'] + activity_id = Activity.ConstructActivityIdFromOperationId(timestamp, message.dict['headers']['op_id']) + message.dict['activity'] = {'activity_id': activity_id, + 'timestamp': timestamp} + callback() + + +class ExtractMD5Hashes(MessageMigrator): + """Migrator that extracts thumbnail and medium MD5 hashes from the + "client_data" field of photo metadata and puts them into standalone + attributes. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.EXTRACT_MD5_HASHES) + + def MigrateForward(self, client, message, callback): + from viewfinder.backend.db.photo import Photo + + for ph_dict in message.dict['photos']: + if 'tn_md5' not in ph_dict: + client_data = ph_dict['client_data'] + ph_dict['tn_md5'] = client_data['tn_md5'] + ph_dict['med_md5'] = client_data['med_md5'] + + callback() + + +class InlineInvalidations(MessageMigrator): + """Migrator that removes the new "inline" attribute in the + query_notifications response for older clients. The activity + attribute moves to the top-level of the notification, and the + "update_seq" attribute is moved to the "activity" section from + the "viewpoint" section. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.INLINE_INVALIDATIONS) + + def MigrateBackward(self, client, message, callback): + from viewfinder.backend.db.photo import Photo + + for notify_dict in message.dict['notifications']: + inline_dict = notify_dict.pop('inline', None) + if inline_dict is not None and 'activity' in inline_dict: + notify_dict['activity'] = inline_dict['activity'] + + callback() + + +class ExtractFileSizes(MessageMigrator): + """Migrator that extracts file (tn/med/full/orig) sizes from the + "client_data" field of photo metadata and puts them into standalone attributes. + Some really old clients do not have said sizes in client_data, we skip those. + We assume that if tn_size is specified, so are the others. This is currently true. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.EXTRACT_FILE_SIZES) + + def MigrateForward(self, client, message, callback): + from viewfinder.backend.db.photo import Photo + + for ph_dict in message.dict['photos']: + if 'client_data' not in ph_dict: + continue + if 'tn_size' not in ph_dict: + client_data = ph_dict['client_data'] + if 'tn_size' in client_data: + ph_dict['tn_size'] = int(client_data.get('tn_size', 0)) + ph_dict['med_size'] = int(client_data.get('med_size', 0)) + ph_dict['full_size'] = int(client_data.get('full_size', 0)) + ph_dict['orig_size'] = int(client_data.get('orig_size', 0)) + + callback() + + +class InlineComments(MessageMigrator): + """Migrator that removes inlined comments from the query_notifications response for older + clients. A comment invalidation is instead created for older clients. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.INLINE_COMMENTS) + + def MigrateBackward(self, client, message, callback): + from viewfinder.backend.db.comment import Comment + + for notify_dict in message.dict['notifications']: + if 'inline' in notify_dict and 'comment' in notify_dict['inline']: + comment_dict = notify_dict['inline'].pop('comment') + start_key = Comment.ConstructCommentId(comment_dict['timestamp'], 0, 0) + notify_dict['invalidate'] = {'viewpoints': [{'viewpoint_id': comment_dict['viewpoint_id'], + 'get_comments': True, + 'comment_start_key': start_key}]} + + callback() + + +class ExtractAssetKeys(MessageMigrator): + """Migrator that extracts asset keys from the 'client_data' field of photo metadata + and puts them into standalone attributes. This is the last use of client_data, so + the field is removed by this migration. + + Unlike the other client_data extractor migrations, the client uses the asset key field + so this migration must be applied in both directions. + + This migration also changes asset keys from a single value to a list. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.EXTRACT_ASSET_KEYS) + + def _FindPhotos(self, message): + # QUERY_EPISODES_RESPONSE + for episode in message.dict.get('episodes', []): + for photo in episode.get('photos', []): + yield photo + + # UPDATE_PHOTO_REQUEST + yield message.dict + + # UPLOAD_EPISODE_REQUEST + for photo in message.dict.get('photos', []): + yield photo + + def MigrateForward(self, client, message, callback): + for photo in self._FindPhotos(message): + client_data = photo.pop('client_data', {}) + if 'asset_key' in client_data: + photo['asset_keys'] = [client_data.pop('asset_key')] + callback() + + def MigrateBackward(self, client, message, callback): + for photo in self._FindPhotos(message): + asset_keys = photo.pop('asset_keys', None) + if asset_keys: + photo.setdefault('client_data', {})['asset_key'] = asset_keys[0] + callback() + + +class SplitNames(MessageMigrator): + """Migrator that splits full names passed to VF auth and update_user methods. Full names + are split into given and family name parts. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.SPLIT_NAMES) + + def MigrateForward(self, client, message, callback): + # Handle VF auth case and update_user case here. + update_dict = message.dict.get('auth_info', message.dict) + + # Only do split if name exists but given_name and family_name do not. + if update_dict and 'name' in update_dict and 'given_name' not in update_dict and 'family_name' not in update_dict: + match = FULL_NAME_RE.match(update_dict['name']) + if match is not None: + update_dict['given_name'] = match.group(1) + if match.group(2): + update_dict['family_name'] = match.group(2) + + callback() + + +class ExplictShareOrder(MessageMigrator): + """Migrator that orders episodes and photos in share requests according to the original + mobile client algorithm for selecting cover photos: + 1) Within share request, oldest to newest episode. + 2) Within episode, newest to oldest photo. + Messages at this version level are expected to be ordered based on the client's intended order for + cover photo selection. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.EXPLICIT_SHARE_ORDER) + + def MigrateForward(self, client, message, callback): + # Sort incoming episodes oldest to newest based on episode_id. + # Episode_ids naturally sort from newest to oldest, so reverse this sort. + message.dict['episodes'].sort(key=lambda episode: episode['new_episode_id'], reverse=True) + # Sort photos from newest to oldest (photo_ids sort this way naturally) + for ep_dict in message.dict['episodes']: + ep_dict['photo_ids'].sort() + + callback() + + +class SuppressBlankCoverPhoto(MessageMigrator): + """Migrator that works around a client bug, in which the client sends a cover photo record + in a share_new request that has a blank photo_id. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPRESS_BLANK_COVER_PHOTO) + + def MigrateForward(self, client, message, callback): + vp_dict = message.dict.get('viewpoint', None) + if vp_dict is not None: + cover_photo_dict = vp_dict.get('cover_photo', None) + if cover_photo_dict is not None: + photo_id = cover_photo_dict.get('photo_id', None) + if not photo_id: + # Remove the entire cover_photo attribute. + del message.dict['viewpoint']['cover_photo'] + + callback() + + +class SupportMultipleIdentitiesPerContact(MessageMigrator): + """Migrator that transforms new format for query_contacts response for downlevel clients.""" + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPORT_MULTIPLE_IDENTITIES_PER_CONTACT) + + def MigrateBackward(self, client, message, callback): + """Migration steps: + * add 'contact_user_id' if first identities entry has user_id property. + * add 'identity' from first identities entry. + * remove 'identities' + * remove 'contact_source' + * remove 'contact_id' + * remove 'labels' if present. + * remove any contacts that have only phone numbers. + * remove any contacts that don't have any identities. + """ + from viewfinder.backend.db.contact import Contact + contacts_list = [] + for contact in message.dict['contacts']: + if 'labels' in contact and Contact.REMOVED in contact['labels']: + continue + if len(contact['identities']) == 0: + continue + first_identity_properties = contact['identities'][0] + if (not first_identity_properties['identity'].startswith('Email:') and + not first_identity_properties['identity'].startswith('FacebookGraph:')): + # Some contacts may have just phone numbers. If there are any email addresses + # or FacebookGraph ids, at least one of them will be the first in the list. + continue + if 'user_id' in first_identity_properties: + contact['contact_user_id'] = first_identity_properties['user_id'] + contact['identity'] = first_identity_properties['identity'] + contact.pop('identities') + contact.pop('contact_source') + contact.pop('contact_id') + assert 'labels' not in contact, 'Migrator should be updated to support labels if present.' + contact.pop('labels', None) + # Add to list because it hasn't been removed. + contacts_list.append(contact) + + # Set new list of contacts which doesn't have 'removed' contacts because we don't want to show + # them to down-level clients. They'll assume they're not present. + message.dict['contacts'] = contacts_list + message.dict['num_contacts'] = len(contacts_list) + + callback() + + +class RenamePhotoLabel(MessageMigrator): + """Migrator that renames the query_episodes photo label from HIDDEN to REMOVED for older + clients. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.RENAME_PHOTO_LABEL) + + def MigrateBackward(self, client, message, callback): + from viewfinder.backend.db.user_post import UserPost + + for ep_dict in message.dict['episodes']: + if 'photos' in ep_dict: + for ph_dict in ep_dict['photos']: + if 'labels' in ph_dict: + labels = set(ph_dict['labels']) + if UserPost.HIDDEN in labels: + labels.remove(UserPost.HIDDEN) + labels.add('removed') + ph_dict['labels'] = list(labels) + + callback() + + +class SuppressAuthName(MessageMigrator): + """Migrator that works around a client bug, in which the client sends a user name to + /link/viewfinder. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPRESS_AUTH_NAME) + + def MigrateForward(self, client, message, callback): + auth_info_dict = message.dict.get('auth_info', None) + if auth_info_dict is not None: + auth_info_dict.pop('name', None) + auth_info_dict.pop('given_name', None) + auth_info_dict.pop('family_name', None) + callback() + + +class SupportRemovedFollowers(MessageMigrator): + """Migrator that extracts and projects only the follower ids from the follower list in a + query_viewpoints response. Later versions of the server return an additional "labels" field + for each follower. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPORT_REMOVED_FOLLOWERS) + + def MigrateBackward(self, client, message, callback): + for vp_dict in message.dict['viewpoints']: + if 'followers' in vp_dict: + vp_dict['followers'] = [foll_dict['follower_id'] for foll_dict in vp_dict['followers']] + callback() + + +class SuppressCopyTimestamp(MessageMigrator): + """Migrator that removes the episode timestamp on incoming share and save operations.""" + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPRESS_COPY_TIMESTAMP) + + def MigrateForward(self, client, message, callback): + for ep_dict in message.dict.get('episodes', []): + ep_dict.pop('timestamp', None) + callback() + + +class SupportContactLimits(MessageMigrator): + """Migrator that truncates and skips various items during upload contacts in order to stay + under limits. + """ + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPORT_CONTACT_LIMITS) + + def MigrateForward(self, client, message, callback): + def _TruncateField(dict, name, limit): + # BUG(Andy): In production, limit is in Unicode UCS-4 chars (Python wide build). Some of + # our dev machines are using a Python narrow build, which means this will be UTF-16 + # codepoints. To fix this, we need to configure all dev machines to use a narrow build. + if len(dict.get(name, '')) > limit: + dict[name] = dict[name][:limit] + + for contact_dict in message.dict.get('contacts', []): + _TruncateField(contact_dict, 'name', 1000) + _TruncateField(contact_dict, 'given_name', 1000) + _TruncateField(contact_dict, 'family_name', 1000) + + _TruncateField(contact_dict, 'identities', 50) + for ident_dict in contact_dict.get('identities', []): + _TruncateField(ident_dict, 'identity', 1000) + _TruncateField(ident_dict, 'description', 1000) + + callback() + + +class SuppressEmptyTitle(MessageMigrator): + """Migrator that removes the viewpoint title from incoming update_viewpoint operations.""" + def __init__(self): + MessageMigrator.__init__(self, Message.SUPPRESS_EMPTY_TITLE) + + def MigrateForward(self, client, message, callback): + if not message.dict.get('title', None): + message.dict.pop('title', None) + callback() + + +REQUIRED_MIGRATORS = [AddHeadersMigrator()] +"""Define list of migrators that *every* message needs to include in +its migration list. The required list can easily be merged with a +message-specific list using a statement like: + + sorted(REQUIRED_MIGRATORS + [MY_MIGRATOR]) +""" + +# ----------------------------------------------------------------- +# Define an instance of each optional migrator so that they can be +# easily included in migration lists. +# ----------------------------------------------------------------- +RENAME_EVENT = RenameEventMigrator() +ADD_TO_VIEWPOINT = AddToViewpointMigrator() +QUERY_EPISODES = QueryEpisodesMigrator() +UPDATE_POST = UpdatePostMigrator() +ADD_OP_HEADER = AddOpHeaderMigrator() +ADD_ACTIVITY = AddActivityMigrator() +EXTRACT_MD5_HASHES = ExtractMD5Hashes() +INLINE_INVALIDATIONS = InlineInvalidations() +EXTRACT_FILE_SIZES = ExtractFileSizes() +INLINE_COMMENTS = InlineComments() +EXTRACT_ASSET_KEYS = ExtractAssetKeys() +SPLIT_NAMES = SplitNames() +EXPLICIT_SHARE_ORDER = ExplictShareOrder() +SUPPRESS_BLANK_COVER_PHOTO = SuppressBlankCoverPhoto() +SUPPORT_MULTIPLE_IDENTITIES_PER_CONTACT = SupportMultipleIdentitiesPerContact() +RENAME_PHOTO_LABEL = RenamePhotoLabel() +SUPPRESS_AUTH_NAME = SuppressAuthName() +SUPPORT_REMOVED_FOLLOWERS = SupportRemovedFollowers() +SUPPRESS_COPY_TIMESTAMP = SuppressCopyTimestamp() +SUPPORT_CONTACT_LIMITS = SupportContactLimits() +SUPPRESS_EMPTY_TITLE = SuppressEmptyTitle() + + +MAX_MESSAGE_VERSION = Message.MAX_VERSION +"""Maximum message version that the server *understands*. However, just +because the server understands this version doesn't mean it will accept +messages from the client that have this version, nor generate messages of +this format when storing into the operations table. See +"SUPPORTED_MESSAGE_VERSION" for more details. When messages having older +formats arrive from the client or are read from the operations table, +they are migrated to this max version so that internal server code only +has to deal with one format. +""" + +MAX_SUPPORTED_MESSAGE_VERSION = Message.SUPPRESS_EMPTY_TITLE +"""Maximum message version that the server *fully supports*. The +supported message version is also guaranteed to be fully rolled out and +supported by *all* other servers. The server will accept messages from +the client that do not exceed this version. In addition, the server will +always generate operation table messages using this format, as it can be +confident that other servers will be able to process this version. Doing +this avoids problems like these: + + - New server stores new operation message. Older server tries to pick + up the operation message in order to run it. + + - New version is rolled out to production. New operation message is saved + in the operations table. New version has problems, and so is rolled + back. Restored server running old version tries to pick up the + operation message in order to run it. +""" + +MIN_SUPPORTED_MESSAGE_VERSION = Message.INITIAL_VERSION +"""Minimum message version that the server understands. If it receives a +message with a version that is less than this version, it will return an +error. This version will be increased as we drop support for older message +formats. +""" diff --git a/backend/base/otp.py b/backend/base/otp.py new file mode 100755 index 0000000..3741f75 --- /dev/null +++ b/backend/base/otp.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python +# +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Creates and verifies one time passwords (OTPs). + +OTPs are implemented as an SHA1 digest of: + - user secret + - GMT (UTC) time rounded to nearest 30s increment + +This module tracks the number of OTP attempts and the specific codes +encountered so it can warn of brute-force or MITM attacks. + +The OTP tokens generated are compatible with Google Authenticator, +and the mobile devices it supports. + +Example Usage: + +For a new administrator, run the following to create a secret for the +admin and get a verification code and QRcode URL for initializing +Google Authenticator: + +% python -m viewfinder.backend.base.otp --otp_mode=new_secret --domain=viewfinder.co --user= + +To set a password for the admin: + +% python -m viewfinder.backend.base.otp --otp_mode=set_pwd --user= + +To display an existing secret, as well as the verification code and +QRcode URL: + +% python -m viewfinder.backend.base.otp --otp_mode=display_secret --user= + + +To get an OTP value for a user: + +% python -m viewfinder.backend.base.otp --otp_mode=get --user= + +To verify an OTP: + +% python -m viewfinder.backend.base.otp --otp_mode=verify --user= --otp= + +To generate random bytes: + +% python -m viewfinder.backend.base.otp --otp_mode=(random,randomb64) --bytes= + + + OTPException: exception for otp verification errors. + + GetOTP(): returns the otp for the requesting user at current time. + VerifyOTP(): verifies an OTP for requesting user. + CreateUserSecret(): creates and persists a user's secret. + CreateRandomBytes(): creates random bytes. + GetPassword(): returns the encrypted user password. + SetPassword(): sets a user password from stdin. + VerifyPassword(): verifies a user password. + + GetAdminOpener(): returns an OpenerDirector for retrieving administrative URLs. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + + +import base64 +import bisect +import cookielib +import getpass +import hashlib +import hmac +import json +import logging +import os +import re +import struct +import sys +import time +import urllib2 + +from Crypto.Protocol.KDF import PBKDF2 +from os.path import expanduser +from tornado import ioloop, options +from viewfinder.backend.base import base_options + +import secrets, util + + +options.define("otp_mode", "get", + help="one of { get, verify, new_secret, set_pwd }") +options.define("otp", None, help="the otp if otp_mode=verify was set") +options.define("user", "", help="username") +options.define("bytes", 128, help="number of bytes to generate") + +_SECRET_BYTES = 10 +_GRANULARITY = 30 +_TIMEOUT = 180 +_VERIFY_MODULUS = 1000 * 1000 +_ATTEMPTS_PER_MIN = 3 + +_PASSWORD_VERSION_MD5 = 0 # md5 with a global "salt" +_PASSWORD_VERSION_PBKDF2 = 1 # pbkdf2 with 10k iterations of sha1 + +_CURRENT_PASSWORD_VERSION = _PASSWORD_VERSION_PBKDF2 + + +# History keeps track of the timestamps of recent login attempts, +# as well as all provided OTP codes. +_history = {} + + +class OTPException(Exception): + """Subclass of exception to communicate error conditions upon + attempted verification of OTP. In particular, too many unsuccesful + OTP entry attempts or repeated tokens. + """ + pass + + +def _ComputeOTP(secret, t): + """Computes the HMAC hash of the user secret, and time (in 30s of + seconds from the epoch). SHA1 is used as the internal digest and + time is packed in big-endian order into an 8 byte string. Four + bytes are extracted from the resulting digest (20 bytes in length + for SHA1) based on an offset computed from the last byte of the + digest % 0xF (e.g. from 0 to 14). The result is adjusted for + negative values and taken modulo _VERIFY_MODULUS to yield a + positive, N-digit OTP, where N = log10(_VERIFY_MODULUS). + """ + h = hmac.new(base64.b32decode(secret), struct.pack('>Q', t), hashlib.sha1) + hash = h.digest() + offset = struct.unpack('B', hash[-1])[0] & 0xF + truncated_hash = struct.unpack('>I', hash[offset:offset + 4])[0] + truncated_hash &= 0x7FFFFFFF + truncated_hash %= _VERIFY_MODULUS + return truncated_hash + + +def _SecretName(user): + """Returns the name of the secret file for the specified user. + """ + return "{0}_otp".format(user) + + +def _PasswordName(user): + """Returns the name of the password file for the specified user. + """ + return "{0}_pwd".format(user) + + +def _GenerateSalt(version): + if version == _PASSWORD_VERSION_MD5: + return "" + elif version == _PASSWORD_VERSION_PBKDF2: + return base64.b64encode(os.urandom(8)) + raise ValueError("unsupported password version") + + +def _HashPassword(password, version, salt): + """Hashes the provided password according to the specified version's policy. + + The result is base32 encoded. + """ + if version == _PASSWORD_VERSION_MD5: + m = hashlib.md5() + m.update(password) + m.update(secrets.GetSecret("cookie_secret")) + hashed = m.digest() + elif version == _PASSWORD_VERSION_PBKDF2: + hashed = PBKDF2(password, base64.b64decode(salt), count=10000) + return base64.b32encode(hashed) + + +def _GetUserSecret(user): + """Returns the user secret by consulting the secrets database.""" + secret = secrets.GetSecret(_SecretName(user)) + if not secret: + raise LookupError("no secret has been created for {0}". + format(user)) + return secret + + +def _UpdateUserHistory(user, t, auth): + """Updates the user history with the specified timestamp and auth + code. Truncates arrays to maximum 30 entries each. Returns a list + of (timestamp, otp) tuples for this user's past accesses. + """ + ts = _history.get(user, [])[-30:] + ts.append((t, auth)) + _history[user] = ts + return ts + + +def _ClearUserHistory(): + """Clears the user history (for testing).""" + _history.clear() + + +def _GetActivationURL(user, secret): + """Generates a URL that displays a QR code on the browser for activating + mobile devices with user secret. + """ + return "https://www.google.com/chart?chs=200x200&chld=M|0&cht=qr&chl=" \ + "otpauth://totp/{0}@www.{1}%3Fsecret%3D{2}".format(user, options.options.domain, secret) + + +def GetOTP(user): + """Gets a new OTP for the specified user by looking up the + user's secret in the secrets database and using it to salt an + MD5 hash of time and username. + """ + return _ComputeOTP(_GetUserSecret(user), + long(time.time() / _GRANULARITY)) + + +def VerifyOTP(user, otp): + """Verifies the provided OTP for the user by comparing it to one + generated right now, with successive checks both going forward and + backwards in time to cover timeout range. This accounts for clock + skew or delay in entering the OTP after fetching it. + """ + timestamp = long(time.time()) + challenge = timestamp / _GRANULARITY + units = _TIMEOUT / _GRANULARITY + + secret = _GetUserSecret(user) + ts = _UpdateUserHistory(user, timestamp, otp) + if len(ts) - bisect.bisect_left(ts, (timestamp - 60,)) > _ATTEMPTS_PER_MIN: + raise OTPException("Too many OTP login attempts for {0} " + "in past minute".format(user)) + if [True for x in ts[:-1] if x[1] == otp]: + raise OTPException("Have already seen OTP {0} for " + "{1}".format(otp, user)) + + for offset in range(-(units - 1) / 2, units / 2 + 1): + if int(otp) == _ComputeOTP(secret, challenge + offset): + return + raise OTPException("Entered OTP invalid") + + +def CreateUserSecret(user): + """Generates a random user secret and stores it to the secrets + database. + """ + secret = base64.b32encode(os.urandom(_SECRET_BYTES)) + secrets.PutSecret(_SecretName(user), secret) + DisplayUserSecret(user) + + +def DisplayUserSecret(user): + """Gets the user secret from the secrets database and displays + it, along with the activation URL and verification code. + """ + secret = _GetUserSecret(user) + print "user secret={0}".format(secret) + print "verification code={0}".format(_ComputeOTP(secret, 0)) + print "activation URL:", _GetActivationURL(user, secret) + + +def CreateRandomBytes(bytes, b64encode=False): + """Generates a string of random bytes.""" + if b64encode: + sys.stdout.write(base64.b64encode(os.urandom(bytes))) + else: + sys.stdout.write(os.urandom(bytes)) + + +def GetPassword(user): + """Returns the encrypted user password from the secrets database.""" + s = secrets.GetSecret(_PasswordName(user)) + try: + return json.loads(s) + except ValueError: + # Pre-json format. + return dict(version=_PASSWORD_VERSION_MD5, hashed=s) + + +def SetPassword(user): + """Accepts a user password as input from stdin and stores it to + the secrets database. The user password is stored as _pwd + and is encrypted using the cookie_secret, defined for the + application to secure cookies. + """ + print "Please enter your password twice to reset:" + pwd = getpass.getpass() + pwd2 = getpass.getpass() + assert pwd == pwd2, 'passwords don\'t match' + version = _CURRENT_PASSWORD_VERSION + salt = _GenerateSalt(version) + hashed = _HashPassword(pwd, version, salt) + data = dict(salt=salt, hashed=hashed, version=version) + secrets.PutSecret(_PasswordName(user), json.dumps(data)) + + +def VerifyPassword(user, cleartext_pwd): + """Encrypts the provided `cleartext_pwd` and compares it to the + encrypted password stored for the user in the secrets DB. + """ + expected = GetPassword(user) + hashed = _HashPassword(cleartext_pwd, expected['version'], + expected.get('salt')) + if hashed != expected['hashed']: + raise OTPException("Entered username/password invalid") + + +def VerifyPasswordCLI(user): + """Command-line interface to VerifyPassword, for testing purposes.""" + print "Please enter your password:" + pwd = getpass.getpass() + result = VerifyPassword(user, pwd) + print "Passwords match" if result else "No match" + + +def _PromptForAdminCookie(user, pwd, otp_entry): + """Prompts the user to enter admin username / password and OTP code. + Synchronously authenticates the user/pwd/otp combination with the + server at www.domain and stores resulting auth cookie(s). + + Returns the new admin cookiejar. + """ + if user is None: + user = raw_input('Please enter admin username: ') + else: + print 'Username: %s' % user + if pwd is None: + pwd = getpass.getpass('Please enter admin password: ') + if otp_entry is None: + otp_entry = int(getpass.getpass('Please enter OTP code: ')) + return user, pwd, otp_entry + + +def GetAdminOpener(host, user=None, pwd=None, otp_entry=None, + cookiejar_path=None): + """Returns an OpenerDirector for retrieving administrative URLs. + Uses stored admin cookies if available, or prompts for authentication + credentials and authenticates with server otherwise. + + Based on reitveld codereview script. + """ + opener = urllib2.OpenerDirector() + opener.add_handler(urllib2.HTTPDefaultErrorHandler()) + opener.add_handler(urllib2.HTTPSHandler()) + opener.add_handler(urllib2.HTTPErrorProcessor()) + # TODO(spencer): remove the HTTP handler when we move to AsyncHTTPSTestCase. + # This is only for testing currently. + opener.add_handler(urllib2.HTTPHandler()) + + if cookiejar_path is None: + cookiejar_path = expanduser('~/.viewfinder_admin_cookie') + cookie_jar = cookielib.MozillaCookieJar(cookiejar_path) + if os.path.exists(cookiejar_path): + try: + cookie_jar.load() + logging.info('loaded admin authentication cookies from %s' % + cookiejar_path) + except: + # Otherwise, bad cookies; clear them. + os.unlink(cookiejar_path) + if not os.path.exists(cookiejar_path): + # Create empty file with correct permissions. + fd = os.open(cookiejar_path, os.O_CREAT, 0600) + os.close(fd) + # Always chmod to be sure. + os.chmod(cookiejar_path, 0600) + opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar)) + + class TornadoXSRFProcessor(urllib2.BaseHandler): + """Add tornado's xsrf headers to outgoing requests.""" + handler_order = urllib2.HTTPCookieProcessor.handler_order + 1 + def http_request(self, request): + cookie_header = request.get_header('Cookie') + if cookie_header is not None and '_xsrf=' in cookie_header: + # We have an xsrf cookie in the cookie jar. Copy it into the X-Xsrftoken header. + request.add_unredirected_header('X-Xsrftoken', re.match('_xsrf=([^;]+)', cookie_header).group(1)) + else: + # No xsrf cookie, so just make one up. (this is currently the expected case because cookielib + # considers our xsrf cookie to be a "session" cookie and doesn't save it) + request.add_unredirected_header('X-Xsrftoken', 'fake_xsrf') + if cookie_header: + request.add_unredirected_header('Cookie', '_xsrf="fake_xsrf"; ' + cookie_header) + else: + request.add_unredirected_header('Cookie', '_xsrf="fake_xsrf"') + return request + https_request = http_request + opener.add_handler(TornadoXSRFProcessor()) + + # Look for admin cookie. If it doesn't exist (or is expired), prompt + # and reauthenticate. + if len(cookie_jar) == 0 or \ + any([c.is_expired() for c in cookie_jar if c.domain == host]): + if user is None or pwd is None or otp_entry is None: + user, pwd, otp_entry = _PromptForAdminCookie(user, pwd, otp_entry) + + from viewfinder.backend.www.admin import admin_api + admin_api.Authenticate(opener, host, user, pwd, otp_entry) + cookie_jar.save() + logging.info('saved admin authentication cookies to %s' % cookiejar_path) + + return opener + + +def main(): + io_loop = ioloop.IOLoop.current() + options.parse_command_line() + + def _OnException(type, value, traceback): + logging.error('failed %s' % options.options.otp_mode, exc_info=(type, value, traceback)) + io_loop.stop() + sys.exit(1) + + def _RunOTPCommand(): + with util.ExceptionBarrier(_OnException): + if options.options.otp_mode == "get": + print GetOTP(options.options.user) + elif options.options.otp_mode == "verify": + print VerifyOTP(options.options.user, options.options.otp) + elif options.options.otp_mode == "new_secret": + CreateUserSecret(options.options.user) + elif options.options.otp_mode == "display_secret": + DisplayUserSecret(options.options.user) + elif options.options.otp_mode == "set_pwd": + SetPassword(options.options.user) + elif options.options.otp_mode == "verify_pwd": + VerifyPasswordCLI(options.options.user) + elif options.options.otp_mode == "random": + CreateRandomBytes(options.options.bytes) + elif options.options.otp_mode == "randomb64": + CreateRandomBytes(options.options.bytes, True) + else: + logging.error("unrecognized mode {0}", options.options.otp_mode) + options.print_help() + io_loop.stop() + + print options.options.domain + + secrets.InitSecrets(shared_only=True, callback=_RunOTPCommand) + io_loop.start() + +if __name__ == "__main__": + main() diff --git a/backend/base/process_util.py b/backend/base/process_util.py new file mode 100644 index 0000000..cedcc55 --- /dev/null +++ b/backend/base/process_util.py @@ -0,0 +1,36 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. +"""Utility functions for unix-style processes. +""" + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +import os +import sys + +_process_name = None + +# http://stackoverflow.com/questions/564695/is-there-a-way-to-change-effective-process-name-in-python +def SetProcessName(newname): + '''Attempts to set the process name (as reported by tools like top). + + Only works on linux. See also the pypi module `setproctitle`, which + is a more robust and portable implementation of this idea. + + Setting the process name to the name of the main script file allows + "pidof -x" (and therefore redhat-style init scripts) to work. + ''' + global _process_name + _process_name = newname + try: + from ctypes import cdll, byref, create_string_buffer + libc = cdll.LoadLibrary('libc.so.6') + buff = create_string_buffer(len(newname) + 1) + buff.value = newname + libc.prctl(15, byref(buff), 0, 0, 0) + except: + pass + +def GetProcessName(): + if not _process_name: + return os.path.basename(sys.argv[0]) + return _process_name diff --git a/backend/base/rate_limiter.py b/backend/base/rate_limiter.py new file mode 100644 index 0000000..5b68018 --- /dev/null +++ b/backend/base/rate_limiter.py @@ -0,0 +1,94 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +""" Strict rate-limiter. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import logging +import math +import time + +from functools import partial +from tornado.ioloop import IOLoop +from viewfinder.backend.base import counters + +class RateLimiter(object): + """Rate limiter allowing no more than the specified qps. + The rate limiter keeps a count of "available requests" that can be issued right away. + Any time ComputeBackoffSecs is called, 'available' is incremented by the qps * time_spent_since_last_call. + + This method strictly enforces rate limiting using a sliding window. + """ + + def __init__(self, qps, unavailable_qps=0.0, qps_counter=None, backoff_counter=None): + """QPS is the number of desired queries per second. 'unavailable_qps' will be subtracted from 'qps'. + If 'qps_counter' is not None, it is incremented when Add() is called. + If 'backoff_counter' is not None, it is incremented by the backoff time in seconds when ComputeBackoffSecs is called + """ + self._qps = qps + self._unavailable_qps = unavailable_qps + self._qps_counter = qps_counter + self._backoff_counter = backoff_counter + + self.available = self._qps - self._unavailable_qps + self.last_time = time.time() + + def _GetQPS(self): + """Return the actual rate-limit we want to use.""" + return self._qps - self._unavailable_qps + + def _Recompute(self): + """Add qps * time_spent to available.""" + now = time.time() + delta = now - self.last_time + # Add the qps freed up since the last call. + limit = self._GetQPS() + self.available += limit * delta + # Make sure the available ops left is in [-qps, +qps]. + # The upper end is throttling (don't exceed the max speed even if we haven't sent anything in > 1s). + # The lower end is to ensure that unexpected "big ops" don't cause us to pause for too long (eg: we may allow + # a Scan with 1 op left, but it may finish with > 1 consumed capacity units). + self.available = max(-limit, min(limit, self.available)) + self.last_time = now + + def Add(self, requests): + """Specify the number of requests issued. Can be negative if correcting for a previous Add().""" + self.available -= requests + if self._qps_counter is not None: + self._qps_counter.increment(requests) + + def SetQPS(self, new_qps): + """Specify a new value for QPS. No need to verify ceilings on 'available', ComputeBackoffSecs will do that.""" + self.available += (new_qps - self._qps) + self._qps = new_qps + + def SetUnavailableQPS(self, new_unavailable_qps): + """Specify a new value for unavailable QPS. No need to verify ceilings on 'available', + ComputeBackoffSecs will do that. + """ + self.available -= (new_unavailable_qps - self._unavailable_qps) + self._unavailable_qps = new_unavailable_qps + + def ComputeBackoffSecs(self): + """Return the number of backoff seconds needed to remain within the desired qps. This should be called only if + the backoff will be done. To check whether backoff is needed without performing it, call NeedsBackoff. + """ + self._Recompute() + if self.available >= 0.0: + # We should technically be checking against 1.0 (since this would mean one request available), but this would + # cause us to send nothing at all when dealing with very small numbers. + return 0.0 + else: + # Time to sleep until we reach positive. |available| / (qps - unavailable) + # The -1 is to reach "available=1.0". The only exception is when we're between 0-1. Otherwise, we need the + # offset to reach the desired rate. + backoff = min(1.0, math.fabs((self.available - 1.0) / self._GetQPS())) + if self._backoff_counter is not None: + self._backoff_counter.increment(backoff) + return backoff + + def NeedsBackoff(self): + """Returns whether or not we will need to backoff. This does not increment the backoff counter.""" + self._Recompute() + return self.available < 0.0 diff --git a/backend/base/retry.py b/backend/base/retry.py new file mode 100644 index 0000000..d85c5f2 --- /dev/null +++ b/backend/base/retry.py @@ -0,0 +1,286 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Reusable functional retry patterns. + +Certain kinds of operations are not always reliable, such as accessing +a network (the network may be down), or communicating with a remote +service (the service may be busy). These operations can fail with +"transient" errors. Given enough time, transient errors may be resolved. +Therefore, the caller may wish to retry the operation several times in +the hope that it will eventually succeed. Callers often want to control +the number of retries, the total amount of time spent retrying, and other +properties of the retry operation. Taken together, these properties +are called a "retry policy". The functions exported by this module +enable a retry policy to be defined and then used to govern the retry +of any arbitrary asynchronous operation. Furthermore, the retry policy +and manager can be sub-classed if custom retry policies are desired. + +Example Usage: + + # Create retry policy that retries 3 times, with no delay between attempts, + # as long as the HTTP response has an error. Typically this policy would + # be created once and reused throughout the application. + http_retry_policy = RetryPolicy(max_tries=3, + check_result=lambda resp: resp.Error) + + # Retry an asynchronous HTTP fetch operation using this policy. + CallWithRetryAsync(http_retry_policy, client.fetch, + 'http://www.google.com', + callback=handle_response) + + # Create retry policy that retries for up to 30 seconds, starting with + # a retry interval of at least 1 second, and exponentially backing off + # to at most 10 seconds between retries. Retry if the operation fails + # with an exception. + retry_policy = RetryPolicy(timeout=timedelta(seconds=30), + min_delay=timedelta(seconds=1), + max_delay=timedelta(seconds=10), + check_exception=lambda typ, val, tb: True) + # or use check_exception=RetryPolicy.AlwaysRetryOnException +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import sys +import random +import functools +import time +import logging +import traceback +import util + +from datetime import timedelta +from tornado.ioloop import IOLoop +from tornado.stack_context import ExceptionStackContext + +class RetryPolicy(object): + """Defines a group of properties that together govern whether and how + an operation will be retried. The base class defines a number of common + retry properties. It can handle two common asynchronous patterns: + + - Errors are never raised as exceptions, but instead passed to the + callback function. Define the "check_result" argument in this case. + - Errors are raised as exceptions, which are expected to be handled + by a stack_context. Define the "check_exception" argument in this + case. + + However, if additional customization is needed, then a custom retry + policy can be created to work with CallWithRetryAsync. To do this, + create a subclass of RetryPolicy as well as RetryManager, and + re-implement the constructor(s), CreateManager, and/or DoRetry. + """ + def __init__(self, max_tries=sys.maxint, timeout=timedelta.max, min_delay=timedelta(seconds=0), + max_delay=timedelta.max, check_result=None, check_exception=None): + """Initialize a default instance of the RetryPolicy, choosing among + the following properties: + + max_tries (int) + Maximum number of tries that will be attempted. + + timeout (timedelta or int or float) + If this amount of time is exceeded, then the operation will not be + retried. This is only checked between attempts. If a number is + provided, then it is interpreted as a number of seconds. + + min_delay (timedelta or int or float) + Minimum delay between attempts. After the first attempt, at least + this amount of time must pass before the second attempt will be + made. For subsequent tries, the delay will exponentially increase, + up to the limit specified in max_delay. If a number is provided, + then it is interpreted as a number of seconds. + + max_delay (timedelta or int or float) + Maximum delay between attempts. It is useful to cap this in order + to guarantee that an attempt will be tried at a minimum frequency, + even after exponential back-off. If a number is provided, then it + is interpreted as a number of seconds. + + check_result (func(*callback_args, **callback_kwargs)) + When an asynchronous operation completes, this function is passed + the arguments to the callback function. Therefore, its signature + should match that of the callback passed to the retry-able function. + The check_result function should return true if the operation has + failed with a retry-able error, or false otherwise. + + check_exception (func(type, value, traceback)) + If the asynchronous function throws an exception, this function is + passed the parts of the exception. The check_exception should return + true if the operation has failed with a retry-able error, or false + otherwise. + + The exponential backoff algorithm helps to prevent "retry storms", in + which many callers are repeatedly and insistently retrying the same + operation. This behavior can compound any existing problem. In addition, + a random factor is added to the backoff time in order to desynchronize + attempts that may have been aligned. + """ + self.max_tries = max_tries + self.timeout = timeout if type(timeout) is timedelta else timedelta(seconds=timeout) + self.min_delay = min_delay if type(min_delay) is timedelta else timedelta(seconds=min_delay) + self.max_delay = max_delay if type(max_delay) is timedelta else timedelta(seconds=max_delay) + self.check_result = check_result + self.check_exception = check_exception + + def CreateManager(self): + """Called by CallWithRetry in order to create a RetryManager which can + track the progress of a particular operation. This method can be overridden + if a custom retry policy is created. + """ + return RetryManager(self) + + @staticmethod + def AlwaysRetryOnException(type, value, traceback): + """Static method to always retry on exceptions.""" + return True + +class RetryManager(object): + """For each kind of RetryPolicy, there should be a corresponding + RetryManager which tracks the retry progress of a particular operation. + The CallWithRetryAsync function will call CreateManager on the + RetryPolicy instance in order to get a manager that it can use to + track the progress of retries by calling the DoRetry method. The + manager may track how many tries have been attempted, how much time + has elapsed, etc. + + When the asynchronous operation has completed, CallWithRetryAsync + will invoke the MaybeRetryOnResult function. If the asynchronous + operation fails with an exception, then CallWithRetryAsync will + invoke the MaybeRetryOnException function. If the invoked function + returns false, then no retry will be attempted. Instead, the original + callback will be invoked (in case of MaybeRetryOnResult), or the + exception will be re-raised (in case of MaybeRetryOnException). + However, if the function returns true, then CallWithRetryAsync + expects it to invoke "retry_func" once the retry should be attempted. + The function should never block the calling thread, but it may + perform a non-blocking wait before invoking "retry_func". + """ + def __init__(self, retry_policy): + """Create a RetryManager that is capable of tracking properties + defined in the RetryPolicy base class. This involves tracking the + number of tries attempted so far, along with whether the timeout + deadline has been exceeded. + """ + self.retry_policy = retry_policy + self._num_tries = 0 + self._deadline = time.time() + retry_policy.timeout.total_seconds() + self._delay = None + + def MaybeRetryOnResult(self, retry_func, *result_args, **result_kwargs): + """This function is called by CallWithRetryAsync once the asynchronous + operation has completed and has invoked its callback function. It + returns true if a retry should be attempted. + """ + def CheckRetry(): + """Retry should be attempted if the result inspector function exists and returns true.""" + return self.retry_policy.check_result and self.retry_policy.check_result(*result_args, **result_kwargs) + + def GetLoggingText(): + """Return text that will be logged if retry is necessary.""" + return '%s returned %s' % (util.FormatFunctionCall(retry_func), + util.FormatArguments(*result_args, **result_kwargs)) + + return self._MaybeRetry(retry_func, CheckRetry, GetLoggingText) + + def MaybeRetryOnException(self, retry_func, type, value, tb): + """This function is called by CallWithRetryAsync if the asynchronous + operation raises an exception. It returns true if a retry should be + attempted. + """ + def CheckRetry(): + """Retry should be attempted if the exception inspector function exists and returns true.""" + return self.retry_policy.check_exception and self.retry_policy.check_exception(type, value, tb) + + def GetLoggingText(): + """Return text that will be logged if retry is necessary.""" + return '%s raised exception %s' % (util.FormatFunctionCall(retry_func), + traceback.format_exception(type, value, tb)) + + return self._MaybeRetry(retry_func, CheckRetry, GetLoggingText) + + def _MaybeRetry(self, retry_func, check_retry_func, log_func): + """Helper function that determines whether a retry should be attempted. + A retry is only attempted if "check_func" returns true. The "log_func" + is invoked if a retry is attempted in order to get text that shows + the context of the retry, and which will be logged. + """ + # Check whether max tries have been exceeded. + self._num_tries += 1 + if self._num_tries >= self.retry_policy.max_tries: + return False + + # Check whether timeout has been exceeded. + if time.time() >= self._deadline: + return False + + # Invoke caller-defined function that determines whether a retry-able error has occurred. + if not check_retry_func(): + return False + + # Retry after delay. + if not self._delay: + self._delay = self.retry_policy.min_delay + else: + self._delay *= 2 + + # Cap delay. + if self._delay > self.retry_policy.max_delay: + self._delay = self.retry_policy.max_delay + + # Add random factor to desynchronize retries, still capped by max delay. + sleep_time = timedelta(seconds=(random.random() + 1) * self._delay.total_seconds()) + if sleep_time > self.retry_policy.max_delay: + sleep_time = self.retry_policy.max_delay + + # Start asynchronous sleep and instruct caller to retry. + logging.getLogger().warning('Retrying function after %.2f seconds: %s' % + (sleep_time.total_seconds(), log_func())) + + if sleep_time.total_seconds() == 0: + IOLoop.current().add_callback(retry_func) + else: + IOLoop.current().add_timeout(sleep_time, retry_func) + + return True + +def CallWithRetryAsync(retry_policy, func, *args, **kwargs): + """This is a higher-order function that wraps an arbitrary asynchronous + function (plus its arguments) in order to add retry functionality. If the + wrapped function completes with an error, then CallWithRetryAsync may call + it again. Pass a "retry_policy" argument that derives from RetryPolicy in + order to control the retry behavior. The retry policy determines whether + the function completed with a retry-able error, and then decides how many + times to retry, and how frequently to retry. + """ + # Validate presence of named "callback" argument. + inner_callback = kwargs.get('callback', None) + assert 'callback' in kwargs, 'CallWithRetryAsync requires a named "callback" argument that is not None.' + + retry_manager = retry_policy.CreateManager() + + # Called when "func" is complete; checks whether to retry the call. + def _OnCompletedCall(*callback_args, **callback_kwargs): + """Called when the operation has completed. Determine whether to retry, + based on the arguments to the callback. + """ + retry_func = functools.partial(func, *args, **kwargs) + if not retry_manager.MaybeRetryOnResult(retry_func, *callback_args, **callback_kwargs): + # If the async operation completes successfully, don't want to retry if continuation code raises an exception. + exception_context.check_retry = False + inner_callback(*callback_args, **callback_kwargs) + + def _OnException(type, value, tb): + """Called if the operation raises an exception. Determine whether to retry + or re-raise the exception, based on the exception details. + """ + if exception_context.check_retry: + retry_func = functools.partial(func, *args, **kwargs) + return retry_manager.MaybeRetryOnException(retry_func, type, value, tb) + + # Replace the callback argument with a callback to _OnCompletedCall which will check for retry. + kwargs['callback'] = _OnCompletedCall + + # Catch any exceptions in order to possibly retry in that case. + exception_context = ExceptionStackContext(_OnException) + exception_context.check_retry = True + with exception_context: + func(*args, **kwargs) diff --git a/backend/base/secrets.py b/backend/base/secrets.py new file mode 100644 index 0000000..9d9dec5 --- /dev/null +++ b/backend/base/secrets.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python +# +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Provides a central mechanism for accessing secrets, such +as private keys, cookie secrets, and authentication service +secrets. + +A particular secret is stored in a file in the --secrets_dir +directory. The secret is accessed using the name of the file. + +Secrets may be encrypted (determined at Init time). If so, the +secrets manager will attempt to get the passphrase, either +by looking it up from AMI (if running with --devbox=False), or +getting it from the user. + +If secrets are not encrypted, they may be read and written without +invoking InitSecrets. If they are encrypted, but InitSecrets is not +invoked, then the fetched contents will be the still-encrypted secret. + +There are two secrets managers. One for shared secrets stored in the +repository and encrypted with the master viewfinder passphrase, the +other for user secrets, stored in ~/.secrets/. + +User secrets are only loaded if --devbox is True. In that case, the +shared secrets manager is loaded lazily if needed (a secret is requested +that does not exist in the user secrets manager). + +With --devbox=False (on AWS), the user secrets manager is never used +and the shared secrets manager is initialized right away. +In such a case, AMI metadata must have been successfully fetched before +calling InitSecrets. 'user-data/passphrase' must be in the fetched metadata. + + InitSecrets(): looks up (in AMI metadata) or prompts for pass phrase. + GetSecret(): returns the data of a named secret. + PutSecret(): writes a new secret the the secrets dir. + ListSecrets(): returns a list of available secrets. + GetCrypter(): get Keyczar Crypter using a named secret. + GetSigner(): get Keyczar Signer using a named secret. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import atexit +import base64 +import getpass +import json +import logging +import os +import stat +import sys +import tempfile + +from Crypto.Cipher import AES +from Crypto.Hash import SHA256 +from keyczar import keyczar, keydata, keyinfo +from tornado import options +from tornado.platform.auto import set_close_exec +from viewfinder.backend.base import ami_metadata, keyczar_dict, base_options +from viewfinder.backend.base.exceptions import CannotReadEncryptedSecretError +from viewfinder.backend.base import base_options # imported for option definitions + +try: + import keyring +except ImportError: + keyring = None + +_tempfile_map = dict() +"""Temporary files for modules which require a file for certificate or key data.""" + +# Secrets manager for user keys (domain=viewfinder.co only) +_user_secrets_manager = None +# Secrets manager for shared keys +_shared_secrets_manager = None + + +class SecretsManager(object): + """Manages secrets subdirectory.""" + # SHA256 Digest size in bytes. + DIGEST_BYTES = 256 / 8 + + # The block size for the cipher object; must be 16, 24, or 32 for AES. + BLOCK_SIZE = 32 + + # The character used for padding--with a block cipher such as AES, + # the value you encrypt must be a multiple of BLOCK_SIZE in + # length. This character is used to ensure that your value is always + # a multiple of BLOCK_SIZE. + PADDING = '{' + + def __init__(self, name, domain, secrets_dir): + """Configures the secrets module to get and write secrets to a + subdirectory of --secrets_dir corresponding to 'domain'. + """ + self._secrets = dict() + self._name = name + self.__secrets_subdir = os.path.join(secrets_dir, domain) + self.__passphrase = None + + def Init(self, can_prompt=True, should_prompt=False, query_twice=False): + """If 'encrypted' is True, a passphrase must be determined. The AMI + user-data is queried first for the secrets pass phrase. If + unavailable, the user is prompted via the console for the + pass-phrase before continuing. If 'encrypted' is True, 'query_twice' + determines whether to ask the user twice for the passphrase for + confirmation. This is done when encrypting files so there is less + chance of a user error causing the file contents to be unretrievable. + """ + passphrase_key = 'user-data/passphrase' + + + def _GetPassphraseFromKeyring(): + """Retrieve the passphrase from keyring. Prompts whether to store it if not found. + If a passphrase was retrieved or stored, save to self.__passphrase and return True. + Passphrase is stored in keyring as ('vf-passphrase', os.getlogin()). + """ + # TODO(marc): store passphrase as a keyczar dict. + + if keyring is None: + print "No keyring found" + return False + + user = os.getlogin() + try: + passphrase = keyring.get_password('%s-vf-passphrase' % self._name, user) + except Exception: + logging.warning("Failed to get %s passphrase from keyring" % self._name) + passphrase = None + + if passphrase is not None: + logging.info("Using %s passphrase from system keyring." % self._name) + self.__passphrase = passphrase + return True + return False + + + def _MaybeStorePassphraseInKeyring(): + if keyring is None: + return + + assert self.__passphrase + answer = raw_input("Store %s passphrase in system keyring? [y/N] " % self._name).strip() + user = os.getlogin() + if answer == "y": + try: + keyring.set_password('%s-vf-passphrase' % self._name, user, self.__passphrase) + except Exception: + logging.warning("Failed to store %s passphrase in keyring" % self._name) + + + def _PromptPassphrase(): + if not can_prompt: + raise CannotReadEncryptedSecretError('passphrase is required but was not provided') + + if _GetPassphraseFromKeyring(): + self._ReadSecrets() + else: + # We did not fetch the passphrase from the keyring, ask for it and maybe store it in the keyring. + if query_twice: + print 'Enter %s passphrase twice for confirmation' % self._name + pp1 = getpass.getpass('passphrase: ') + pp2 = getpass.getpass('passphrase: ') + assert pp1 == pp2, 'passphrases don\'t match' + self.__passphrase = pp1 + else: + self.__passphrase = getpass.getpass('%s passphrase: ' % self._name) + + # Make sure the passphrase is correct before trying to store it in the keyring. + self._ReadSecrets() + _MaybeStorePassphraseInKeyring() + + + def _GetPassphrase(): + if options.options.passphrase: + # Passphrase passed as command-line option. + self.__passphrase = options.options.passphrase + self._ReadSecrets() + elif options.options.passphrase_file: + # Passphrase contained in a file + filename = os.path.expanduser(os.path.expandvars(options.options.passphrase_file)) + self.__passphrase = open(filename, 'r').read().strip() + self._ReadSecrets() + elif not options.options.devbox: + # Passphrase is available in AMI metadata. + metadata = ami_metadata.GetAMIMetadata() + if passphrase_key not in metadata: + raise CannotReadEncryptedSecretError('failed to fetch passphrase from AWS instance metadata; ' + 'if running on dev box, use the --devbox option') + self.__passphrase = metadata[passphrase_key] + self._ReadSecrets() + else: + # Prompt for passphrase at command-line. + _PromptPassphrase() + + if should_prompt: + # should_prompt is true, so prompt for the passphrase even if we already have it. + assert can_prompt, 'if should_prompt is true, then can_prompt must also be true' + _PromptPassphrase() + + # Try to read the secrets with no passphrase. This will work if the secrets are not + # encrypted (e.g. test secrets). + try: + self._ReadSecrets() + # Read succeeded, so initialization is complete. + except CannotReadEncryptedSecretError: + # Must get passphrase from command-line, AMI metadata, or by prompting for it. + _GetPassphrase() + + def InitForTest(self): + """Reads the secrets with assumption they are not encrypted.""" + self._need_passphrase = False + self._ReadSecrets() + + def ListSecrets(self): + """Returns a list of available secrets.""" + return self._secrets.keys() + + def HasSecret(self, secret): + """Returns true if the secret is in the secrets map.""" + return secret in self._secrets + + def GetSecret(self, secret): + """Returns the secret from the secrets map.""" + return self._secrets[secret].strip() + + def PutSecret(self, secret, secret_value): + """Writes the secrets file and possibly encrypts the value.""" + self._secrets[secret] = secret_value.strip() + fn = self._GetSecretFile(secret, verify=False) + with open(fn, 'w') as f: + os.chmod(fn, stat.S_IRUSR | stat.S_IWUSR) + if self.__passphrase: + encrypted_secret = self._EncryptSecret(self._secrets[secret]) + f.write(json.dumps(encrypted_secret)) + else: + f.write(self._secrets[secret]) + + def GetCrypter(self, secret): + """Assumes the secret is a Keyczar crypt keyset. Loads the secret + value and returns a Keyczar Crypter object already initialized with + the keyset value. + """ + return keyczar.Crypter(keyczar_dict.DictReader(self.GetSecret(secret))) + + def GetSigner(self, secret): + """Assumes the secret is a Keyczar signing keyset. Loads the secret + value and returns a Keyczar Signer object already initialized with + the keyset value. + """ + return keyczar.Signer(keyczar_dict.DictReader(self.GetSecret(secret))) + + def _GetSecretFile(self, secret, verify=True): + """Concatenates the secret name with the --secrets_dir command + line flag. + """ + path = os.path.join(self.__secrets_subdir, secret) + if verify and not os.access(path, os.R_OK): + raise IOError('unable to access {0}'.format(path)) + return path + + def _ReadSecret(self, secret): + """Reads the secrets file and possibly decrypts it.""" + with open(self._GetSecretFile(secret), 'r') as f: + contents = f.read() + try: + (cipher, ciphertext) = json.loads(contents) + if cipher != 'AES': + # Contents are not in our encryption format, so assume they're not encrypted. + return contents + except: + # Contents are not in legal JSON format, so assume they're not encrypted. + return contents + + return self._DecryptSecret(cipher, ciphertext) + + def _ReadSecrets(self): + """Reads all secrets from the secrets subdir. + """ + try: + secrets = os.listdir(self.__secrets_subdir) + except Exception: + return + for secret in secrets: + self._secrets[secret] = self._ReadSecret(secret) + + def _DecryptSecret(self, cipher, ciphertext): + """Decrypts the ciphertext secret, splits it into the first + DIGEST_BYTES bytes (sha256 message digest), and verifies the digest + matches the secret. Returns the plaintext secret on success. + """ + if not self.__passphrase: + raise CannotReadEncryptedSecretError('no passphrase initialized') + assert cipher == 'AES', 'cipher %s not supported' % cipher + aes_cipher = AES.new(self._PadText(self.__passphrase)) + plaintext = aes_cipher.decrypt(base64.b64decode(ciphertext)).rstrip(SecretsManager.PADDING) + sha256_digest = plaintext[:SecretsManager.DIGEST_BYTES] + plaintext_secret = plaintext[SecretsManager.DIGEST_BYTES:] + sha256 = SHA256.new(plaintext_secret) + assert sha256.digest() == sha256_digest, 'secret integrity compromised: sha256 hash does not match' + return plaintext_secret + + def _EncryptSecret(self, plaintext_secret): + """Computes a SHA256 message digest of the secret, prepends the + digest, pads to a multiple of BLOCK_SIZE, encrypts using an AES + cipher, and base64 encodes. Returns a tuple containing the cipher + used and the base64-encoded, encrypted value. + """ + aes_cipher = AES.new(self._PadText(self.__passphrase)) + sha256 = SHA256.new(plaintext_secret) + assert len(sha256.digest()) == SecretsManager.DIGEST_BYTES, \ + 'expected length of sha256 message digest not 256 bits' + plaintext = self._PadText(sha256.digest() + plaintext_secret) + ciphertext = base64.b64encode(aes_cipher.encrypt(plaintext)) + return ('AES', ciphertext) + + def _PadText(self, text): + """Pads the provided text so that it is a multiple of BLOCK_SIZE. + The padding character is specified by PADDING. Returns the padded + version of 'text'. + """ + if len(text) in (16, 24, 32): + return text + return text + (SecretsManager.BLOCK_SIZE - + len(text) % SecretsManager.BLOCK_SIZE) * SecretsManager.PADDING + + +def GetSharedSecretsManager(can_prompt=None): + """Returns the shared secrets manager. Creates it from options if None. + If can_prompt is None, determine automatically. + """ + global _shared_secrets_manager + if _shared_secrets_manager is None: + _shared_secrets_manager = SecretsManager('shared', options.options.domain, options.options.secrets_dir) + prompt = can_prompt if can_prompt is not None else sys.stderr.isatty() + _shared_secrets_manager.Init(can_prompt=prompt) + return _shared_secrets_manager + + +def GetUserSecretsManager(can_prompt=None): + """Returns the user secrets manager. Creates it from options if None. + If can_prompt is None, determine automatically. + Fails in --devbox=False mode. + """ + assert options.options.devbox, 'User secrets manager is only available in --devbox mode.' + + global _user_secrets_manager + if _user_secrets_manager is None: + # Create the user secrets manager. + _user_secrets_manager = SecretsManager('user', options.options.domain, options.options.user_secrets_dir) + prompt = can_prompt if can_prompt is not None else sys.stderr.isatty() + _user_secrets_manager.Init(can_prompt=prompt) + return _user_secrets_manager + + +def GetSecretsManagerForSecret(secret): + """Returns the appropriate secrets manager for a secret. + If we have a user secrets manager and it holds this secret, return it, otherwise use the shared secrets manager. + The user secrets manager is always initialized (if needed). + """ + global _user_secrets_manager + if _user_secrets_manager is not None and _user_secrets_manager.HasSecret(secret): + return _user_secrets_manager + return GetSharedSecretsManager() + + +def InitSecrets(callback=None, shared_only=False, can_prompt=True): + """Init secrets. + If running with --devbox, initialize the user secrets manager only (the shared secrets manager will be initialized + lazily if needed). + If --devbox is False and shared_only=False, only initialize the shared secrets manager. + shared_only=True should only be used when we know the secrets should NOT be stored in the user secrets (eg: OTP). + """ + if options.options.devbox and not shared_only: + GetUserSecretsManager() + else: + GetSharedSecretsManager() + if callback is not None: + callback() + + +def InitSecretsForTest(): + """Init secrets for test. We only use the shared secrets manager.""" + GetSharedSecretsManager(can_prompt=False) + + +def GetSecret(secret): + """Fetched the named secret.""" + return GetSecretsManagerForSecret(secret).GetSecret(secret) + +def GetSecretFile(secret): + """Fetches the named secret into a temporary file for use with + modules requiring the contents be accessible via a named file (e.g. + Python SSL for keys and certificates). + """ + if sys.platform.startswith('linux'): + # Linux-specific implementation: use an unnamed tempfile, which + # will cease to exist when this process does. Use /dev/fd to get + # a name for the file. + # Note that other platforms (including Mac) have /dev/fd as well, + # but its semantics are different (all copies of a /dev/fd + # file share one seek position, and that position is not reset on + # open), so it's only safe to use on linux. + if secret not in _tempfile_map: + f = tempfile.TemporaryFile() + set_close_exec(f.fileno()) + f.write(GetSecret(secret)) + f.flush() + _tempfile_map[secret] = f + + return '/dev/fd/%d' % _tempfile_map[secret].fileno() + else: + # Default implementation: use a normal named tempfile, and delete + # it when possible with atexit. + if secret not in _tempfile_map: + _, name = tempfile.mkstemp() + with open(name, 'w') as f: + f.write(GetSecret(secret)) + _tempfile_map[secret] = name + atexit.register(os.remove, name) + + return _tempfile_map[secret] + + +def PutSecret(secret, secret_value): + """Writes the specified secret value to a file in the secrets + directory named `secret`. + """ + GetSecretsManagerForSecret(secret).PutSecret(secret, secret_value) + + +def GetCrypter(secret): + """Returns the Keyczar Crypter object returned by the secrets manager + instance GetCrypter method.""" + return GetSecretsManagerForSecret(secret).GetCrypter(secret) + + +def GetSigner(secret): + """Returns the Keyczar Signer object returned by the secrets manager + instance GetSigner method.""" + return GetSecretsManagerForSecret(secret).GetSigner(secret) + + +def CreateCryptKeyset(name): + """Returns a Keyczar keyset to be used for encryption and decryption. + 'name' is the name of the keyset. The keyset is returned as a Python + dict in the format described in the keyczar_dict.py header. + """ + return _CreateKeyset(name, keyinfo.DECRYPT_AND_ENCRYPT, keyinfo.AES) + + +def CreateSigningKeyset(name): + """Returns a Keyczar keyset to be used for signing and signature + verification. 'name' is the name of the keyset. The keyset is + returned as a Python dict in the format described in the + keyczar_dict.py header. + """ + return _CreateKeyset(name, keyinfo.SIGN_AND_VERIFY, keyinfo.HMAC_SHA1) + + +def _CreateKeyset(name, purpose, key_type): + """Constructs a Keyczar keyset, passing the specified arguments to the + KeyMetadata constructor. Adds one primary key to the keyset and returns + the keyset as a Python dict. + """ + # Construct the metadata and add the first crypt key with primary status, meaning + # it will be used to both encrypt/sign and decrypt/verify (rather than just + # decrypt/verify). + meta = keydata.KeyMetadata(name, purpose, key_type) + writer = keyczar_dict.DictWriter() + writer.WriteMetadata(meta) + czar = keyczar.GenericKeyczar(keyczar_dict.DictReader(writer.dict)) + czar.AddVersion(keyinfo.PRIMARY) + czar.Write(writer) + return writer.dict diff --git a/backend/base/secrets_tool.py b/backend/base/secrets_tool.py new file mode 100755 index 0000000..489505b --- /dev/null +++ b/backend/base/secrets_tool.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Command-line tool for creating and encrypting secrets using the +secrets_manager module. + +% python -m viewfinder.backend.base.secrets_tool \ + --secrets_mode={list_secrets, encrypt_secrets, get_secret, + put_secret, put_crypt_keyset} +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import json +import logging +import sys + +from tornado import ioloop, options +from viewfinder.backend.base import base_options # imported for option definitions +from viewfinder.backend.base import secrets, util + + +options.define('secrets_mode', 'list_secrets', + help='mode for command line operation; see help text in module') + +options.define('secret', '', help='name of the secret to put or get') + +options.define('shared', default=True, + help='work on the shared secrets manager. If false, use the user secrets manager') + + +def _GetSecretsManager(): + if options.options.shared: + return secrets.GetSharedSecretsManager() + else: + return secrets.GetUserSecretsManager() + + +def _ListSecrets(io_loop): + """Lists all secrets.""" + for f in _GetSecretsManager().ListSecrets(): + print ' %s' % f + io_loop.stop() + + +def _GetSecret(io_loop, secret): + """Get a secret by name and output to stdout.""" + print '%s:\n%s' % (secret, _GetSecretsManager().GetSecret(secret)) + io_loop.stop() + + +def _PutSecret(io_loop, secret): + """Reads the new secret from stdin and writes to secrets subdir.""" + _GetSecretsManager().PutSecret(secret, sys.stdin.read()) + io_loop.stop() + + +def _PutCryptKeyset(io_loop, secret): + """Creates a new Keyczar crypt keyset used for encryption and decryption + and writes it to secrets subdir.""" + _GetSecretsManager().PutSecret(secret, json.dumps(secrets.CreateCryptKeyset(secret))) + io_loop.stop() + + +def _PutSigningKeyset(io_loop, secret): + """Creates a new Keyczar crypt keyset used for signing and signature + verification and writes it to secrets subdir.""" + _GetSecretsManager().PutSecret(secret, json.dumps(secrets.CreateSigningKeyset(secret))) + io_loop.stop() + + +def _EncryptSecrets(io_loop): + """Lists all secrets files and encrypts each in turn. The passphrase + for encryption is solicited twice for confirmation. + """ + print 'Initializing existing secrets manager...' + ex_sm = _GetSecretsManager() + + print 'Initializing new secrets manager...' + if options.options.shared: + new_sm = secrets.SecretsManager('shared', options.options.domain, options.options.secrets_dir) + else: + new_sm = secrets.SecretsManager('user', options.options.domain, options.options.user_secrets_dir) + new_sm.Init(should_prompt=True, query_twice=True) + + print 'Encrypting secrets...' + for secret in ex_sm.ListSecrets(): + print ' %s' % secret + new_sm.PutSecret(secret, ex_sm.GetSecret(secret)) + io_loop.stop() + + +def main(): + """Parses command line options and, if directed, executes some operation + to transform or create secrets from the command line. + """ + io_loop = ioloop.IOLoop.current() + options.parse_command_line() + + def _OnException(type, value, traceback): + logging.error('failed %s' % options.options.secrets_mode, exc_info=(type, value, traceback)) + io_loop.stop() + sys.exit(1) + + with util.ExceptionBarrier(_OnException): + if options.options.secrets_mode == 'list_secrets': + _ListSecrets(io_loop) + elif options.options.secrets_mode == 'get_secret': + _GetSecret(io_loop, options.options.secret) + elif options.options.secrets_mode == 'put_secret': + _PutSecret(io_loop, options.options.secret) + elif options.options.secrets_mode == 'put_crypt_keyset': + _PutCryptKeyset(io_loop, options.options.secret) + elif options.options.secrets_mode == 'put_signing_keyset': + _PutSigningKeyset(io_loop, options.options.secret) + elif options.options.secrets_mode == 'encrypt_secrets': + _EncryptSecrets(io_loop) + else: + raise Exception('unknown secrets_mode: %s' % options.options.secrets_mode) + + io_loop.start() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/backend/base/statistics.py b/backend/base/statistics.py new file mode 100644 index 0000000..2d4bad1 --- /dev/null +++ b/backend/base/statistics.py @@ -0,0 +1,89 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" +Functions to pretty print basic statistics and histograms. + +Print basic statistics about a list of numbers: +>>> a = [1, 1, 2, 3, 4, 4] +>>> print statistics.FormatStats(a, percentiles=[90, 95, 99], indent=2) + mean=2.50 + median=2.50 + stddev=1.26 + 50/90/95/99 percentiles=[2.5, 4.0, 4.0, 4.0] + +Pretty print a histogram +>>> a = [ 1, 1, 2, 3, 4, 4] +>>> print statistics.HistogramToASCII(a, bins=5, indent=2) + [1-1) 2 33.33% ############################################################## + [1-2) 1 16.67% ############################### + [2-3) 1 16.67% ############################### + [3-4] 2 33.33% ############################################################## +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +try: + import numpy +except ImportError: + numpy = None +import pprint +import string + +def FormatStats(data, percentiles=None, indent=0): + """ Compute basic stats for an array and return a string + containing average, median, standard deviation and + percentiles (if not None). indent specifies the number + of leading spaces on each line. + """ + if len(data) == 0: return '' + leader = ' ' * indent + out_str = leader + 'mean=%.2f' % numpy.mean(data) + out_str += '\n' + leader + 'median=%.2f' % numpy.median(data) + out_str += '\n' + leader + 'stddev=%.2f' % numpy.std(data) + if percentiles: + out_str += '\n' + leader + '/'.join(map(str, percentiles)) + out_str += ' percentiles=%s' % numpy.percentile(data, percentiles) + return out_str + +def HistogramToASCII(data, bins=10, line_width=80, indent=0): + """ Compute the histogram for 'data' and generate its string + representation. line_width is used to compute the maximum bar + length. indent specifies the number of leading spaces on each line. + """ + if len(data) == 0: return '' + hist, buckets = numpy.histogram(data, bins=bins) + percent_multiplier = float(100) / hist.sum() + + # buckets contains the bucket limits. it therefore has one more element + # than hist. the last bin is closed. all others half-open. + bin_edge_length = len(str(int(buckets[-1]))) + num_length = len(str(hist.max())) + + bar_max = line_width - indent - bin_edge_length * 2 - num_length + # spaces, brackets, percentage, percent and dash. and one at the end. + bar_max -= 13 + bar_divider = float(hist.max()) / bar_max + + # closing character for ranges. most are half-open: ')' + closing_char = ')' + out_str = '' + for i in range(len(hist)): + last = (i == len(hist) - 1) + if last: + # last bucket is closed. + closing_char = ']' + line_str = ' ' * indent + # add bucket description with zfilled range limits to have equal string length. + line_str += '[' + string.zfill(int(buckets[i]), bin_edge_length) + line_str += '-' + string.zfill(int(buckets[i+1]), bin_edge_length) + closing_char + # count of items in this bucket and percentage of total. + line_str += ' ' + string.zfill(hist[i], num_length) + line_str += ' ' + '%5.2f%%' % (hist[i] * percent_multiplier) + # histogram bar, but only if we have room to display it. + if bar_max > 0: + line_str += ' ' + '#' * (hist[i] / bar_divider) + + out_str += line_str + if not last: + out_str += '\n' + return out_str diff --git a/backend/base/test/__init__.py b/backend/base/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/base/test/base64hex_test.py b/backend/base/test/base64hex_test.py new file mode 100644 index 0000000..ec4cde4 --- /dev/null +++ b/backend/base/test/base64hex_test.py @@ -0,0 +1,80 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Test encode/decode & maintenance of sort ordering for +base64hex encoding. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import random +import unittest + +from viewfinder.backend.base import base64hex + +class Base64HexTestCase(unittest.TestCase): + def _RandomString(self): + """Use a multiple of 6 for the length of the random string to ensure + no padding is used in the encoded result. + """ + length = random.randint(1, 10) * 6 + return ''.join([chr(random.randint(0,255)) for i in xrange(length)]) + + def testEncodeDecode(self): + num_trials = 1000 + for i in xrange(num_trials): + s = self._RandomString() + enc = base64hex.B64HexEncode(s) + dec = base64hex.B64HexDecode(enc) + self.assertEqual(s, dec) + + def testSortOrder(self): + num_trials = 1000 + for i in xrange(num_trials): + s1 = self._RandomString() + s2 = self._RandomString() + enc1 = base64hex.B64HexEncode(s1) + enc2 = base64hex.B64HexEncode(s2) + assert (s1 < s2) == (enc1 < enc2), 's1: %s, s2: %s, enc1: %s, enc2: %s' % (s1, s2, enc1, enc2) + + def testInvalidDecode(self): + # Bad character + self.assertRaises(TypeError, base64hex.B64HexDecode, '@') + # Padding unnecessary. + self.assertRaises(TypeError, base64hex.B64HexDecode, 'RV_3SDFO=') + # Wrong amount of padding. + self.assertRaises(TypeError, base64hex.B64HexDecode, 'RV_3SDFO==') + + def testKnownValues(self): + # random strings of lengths 0-19 + data = [ + ('', ''), + ('\xf9', 'yF=='), + ('*\xc9', '9gZ='), + ('T\xe7`', 'KDSV'), + ('\xd2\xe9H\x0c', 'oi_72-=='), + ('K\x84\x03\xeb\xe8', 'HsF2uyV='), + ('\xebl\xe5\xa3\xa3\xf8', 'uqn_cuEs'), + ('\x04\x88yR\xef\xa1M', '07WtJiyWIF=='), + ('h\x8c\xa2\xb8h\x8c\x19v', 'P7mXi5XB5MN='), + ('\x06\xc7_M\x19$\x88v\xb4', '0gSUIGZZX6Po'), + ('\x1d\xab\xefI\xf7\x7fY\xa4\r\xe8', '6PjjHUSzLPFCu-=='), + ('\xa4=\xe6\x1b\x00\xb1\r\xba\xcc\xca\xf4', 'd2ra5k1l2QfBmjF='), + ('\xd7\xac\xa8\x97\xc2\x14\x16)\xf5"\xc8d', 'pumc_w7J4Xbp7gWZ'), + ('\xab\xb3%\xd3&I\xfd\x9cc\x91\x17\xd7\xdf', 'evB_omO8zOlYZGUMrk=='), + ('O\x8dO\xf4\nd\xc4\xf5W]\xdf\xd3\xa9\xfe', 'IspEx-dZlEKMMSzIeUs='), + ("\xca\xdc\x8d'\xf0\xc5a\x93b\x1c@4\xdaC\x9a", 'mhmC8z24NOCX63-oqZDP'), + ('\xe1\x00\xf7\xd8p\xef\x08v\xca\x9b\x81INPvu', 'sF2rq62j16Q9as48I_0qSF=='), + ('8_\xaeS\xb9\xa9\xb7\x1e\x99\x8c\x06\xc7\xa9\xa2F\xb5\x0f', 'D4yiJvadhluOY-Q6eP85hFw='), + ('"\xc3)!J H\x98\ro\xe1\\\xc2a\xc9\xe2v\xe8', '7gBd7JcVH8VCQy4Rka68sbQc'), + ('\xe3\xb1?_\x97a<\xc5\xf5Cj\x86\xbeB\xc3F\xcc\x1ai', 'sv3zMtSWEBMpFqe5jZA2GgkPPF=='), + ] + for s, expected in data: + try: + self.assertEqual(base64hex.B64HexEncode(s), expected) + self.assertEqual(base64hex.B64HexDecode(expected), s) + self.assertEqual(base64hex.B64HexEncode(s, padding=False), expected.rstrip('=')) + self.assertEqual(base64hex.B64HexDecode(expected.rstrip('='), padding=False), s) + except: + logging.info("failed on %r", (s, expected)) + raise diff --git a/backend/base/test/client_version_test.py b/backend/base/test/client_version_test.py new file mode 100644 index 0000000..336283f --- /dev/null +++ b/backend/base/test/client_version_test.py @@ -0,0 +1,68 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Test versioning. +""" + +import unittest + +from viewfinder.backend.base.client_version import ClientVersion + + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +class ClientVersionTestCase(unittest.TestCase): + def testParsing(self): + version = ClientVersion(None) + self.assertFalse(version.IsValid()) + version = ClientVersion('') + self.assertFalse(version.IsValid()) + version = ClientVersion('1') + self.assertFalse(version.IsValid()) + + version = ClientVersion('1.3.0') + self.assertTrue(version.IsValid()) + self.assertFalse(version.IsDev()) + self.assertFalse(version.IsTestFlight()) + self.assertTrue(version.IsAppStore()) + + version = ClientVersion('1.3.0.dev') + self.assertTrue(version.IsValid()) + self.assertTrue(version.IsDev()) + self.assertFalse(version.IsTestFlight()) + self.assertFalse(version.IsAppStore()) + + version = ClientVersion('1.3.0.adhoc') + self.assertTrue(version.IsValid()) + self.assertFalse(version.IsDev()) + self.assertTrue(version.IsTestFlight()) + self.assertFalse(version.IsAppStore()) + + def testCompare(self): + version = ClientVersion('1.6.0.40') + + self.assertTrue(version.LT('1.7')) + self.assertTrue(version.LT('1.6.1')) + self.assertTrue(version.LT('1.6.0.41')) + self.assertFalse(version.LT('1.6.0.40')) + self.assertFalse(version.LT('1.6')) + + self.assertTrue(version.LE('1.7')) + self.assertTrue(version.LE('1.6.1')) + self.assertTrue(version.LE('1.6.0.41')) + self.assertTrue(version.LE('1.6.0.40')) + self.assertFalse(version.LE('1.6')) + + self.assertFalse(version.EQ('1.6.0')) + self.assertTrue(version.EQ('1.6.0.40')) + + self.assertTrue(version.GT('1.5')) + self.assertTrue(version.GT('1.6.0')) + self.assertTrue(version.GT('1.6.0.39')) + self.assertFalse(version.GT('1.6.0.40')) + self.assertFalse(version.GT('1.6.0.41')) + + self.assertTrue(version.GE('1.5')) + self.assertTrue(version.GE('1.6.0')) + self.assertTrue(version.GE('1.6.0.39')) + self.assertTrue(version.GE('1.6.0.40')) + self.assertFalse(version.GE('1.6.0.41')) diff --git a/backend/base/test/context_local_test.py b/backend/base/test/context_local_test.py new file mode 100644 index 0000000..9f0d4e2 --- /dev/null +++ b/backend/base/test/context_local_test.py @@ -0,0 +1,76 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for ContextLocal object.""" + +from __future__ import with_statement + +__author__ = 'Matt Tracy (matt@emailscrubbed.com)' + +import unittest +from functools import partial +from tornado.stack_context import StackContext +from viewfinder.backend.base import context_local, util, testing + +class ExampleContext(context_local.ContextLocal): + def __init__(self, val): + super(ExampleContext, self).__init__() + self.val = val + +class ExampleContextTwoParams(context_local.ContextLocal): + def __init__(self, val1, val2): + super(ExampleContextTwoParams, self).__init__() + self.val1 = val1 + self.val2 = val2 + +class ContextLocalTestCase(testing.BaseTestCase): + """Tests using a few basic ContextLocal subclasses.""" + + def testNestedContexts(self): + """Test the nesting of a single ContextLocal subclass.""" + with util.Barrier(self._OnSuccess, on_exception=self._OnException) as b: + with StackContext(ExampleContext(1)): + self.io_loop.add_callback(partial(self._VerifyExampleContext, 1, b.Callback())) + with StackContext(ExampleContext(2)): + self._VerifyExampleContext(2, util.NoCallback) + self.io_loop.add_callback(partial(self._VerifyExampleContext, 2, b.Callback())) + self._VerifyExampleContext(1, util.NoCallback) + + self.wait() + + def testMultipleContextTypes(self): + """Test the usage of multiple ContextLocal subclasses in tandem.""" + with util.Barrier(self._OnSuccess, on_exception=self._OnException) as b: + with StackContext(ExampleContext(1)): + with StackContext(ExampleContextTwoParams(2, 3)): + self._VerifyExampleContext(1, util.NoCallback) + self._VerifyExampleContextTwoParams(2, 3, util.NoCallback) + self.io_loop.add_callback(partial(self._VerifyExampleContext, 1, b.Callback())) + self.io_loop.add_callback(partial(self._VerifyExampleContextTwoParams, 2, 3, b.Callback())) + + self.wait() + + + def _OnSuccess(self): + self.assertTrue(ExampleContext.current() is None, "Unexpected example context: context") + self.assertTrue(ExampleContextTwoParams.current() is None) + self.stop() + + def _OnException(self, type, value, traceback): + try: + raise + finally: + self.stop() + + def _VerifyExampleContext(self, expected, callback): + self.assertEqual(ExampleContext.current().val, expected) + callback() + + def _VerifyExampleContextTwoParams(self, expected1, expected2, callback): + self.assertEqual(ExampleContextTwoParams.current().val1, expected1) + self.assertEqual(ExampleContextTwoParams.current().val2, expected2) + callback() + + + + + diff --git a/backend/base/test/counters_test.py b/backend/base/test/counters_test.py new file mode 100644 index 0000000..c880033 --- /dev/null +++ b/backend/base/test/counters_test.py @@ -0,0 +1,158 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +""" Tests for the viewfinder counters module. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import time +from viewfinder.backend.base import counters +import unittest + + +class CountersTest(unittest.TestCase): + """Test for the various counter types, the CounterManager and Meter.""" + def testTotal(self): + """Test for the Total counter type.""" + total = counters._TotalCounter('mytotal', 'Description') + sampler = total.get_sampler() + sampler2 = total.get_sampler() + + self.assertEqual(0, sampler()) + + total.increment() + self.assertEqual(1, sampler()) + + total.increment(4) + self.assertEqual(5, sampler()) + + total.decrement() + self.assertEqual(4, sampler()) + + total.decrement(5) + self.assertEqual(-1, sampler()) + self.assertEqual(-1, sampler2()) + + def testDelta(self): + """Test for the delta counter type.""" + delta = counters._DeltaCounter('mydelta', 'Description') + sampler = delta.get_sampler() + sampler2 = delta.get_sampler() + + self.assertEqual(0, sampler()) + + delta.increment() + self.assertEqual(1, sampler()) + + delta.increment(4) + self.assertEqual(4, sampler()) + + delta.decrement() + self.assertEqual(-1, sampler()) + + delta.decrement(5) + self.assertEqual(-5, sampler()) + self.assertEqual(-1, sampler2()) + + def testAverage(self): + avg = counters._AverageCounter('myaverage', 'Description') + sampler = avg.get_sampler() + sampler2 = avg.get_sampler() + + self.assertEqual(0, sampler()) + + avg.add(5) + avg.add(10) + avg.add(15) + self.assertEqual(10, sampler()) + + avg.add(6) + avg.add(12) + avg.add(18) + self.assertEqual(12, sampler()) + + self.assertEqual(0, sampler()) + + self.assertEqual(11, sampler2()) + + def testRate(self): + time_val = [0] + def test_time(): + time_val[0] += 1 + return time_val[0] + + rate = counters._RateCounter('myrate', 'description', time_func=test_time) + sampler = rate.get_sampler() + sampler2 = rate.get_sampler() + + self.assertEqual(0, sampler()) + + rate.increment(5) + rate.increment(10) + rate.increment(15) + self.assertEqual(30, sampler()) + + rate.increment(5) + rate.increment(10) + rate.increment(15) + test_time() + self.assertEqual(15, sampler()) + self.assertEqual(12, sampler2()) + + rate = counters._RateCounter('myrate', 'description', 60, time_func=test_time) + sampler = rate.get_sampler() + self.assertEqual(0, sampler()) + + rate.increment(5) + rate.increment(10) + rate.increment(15) + self.assertEqual(1800, sampler()) + + rate.increment(5) + rate.increment(10) + rate.increment(15) + test_time() + self.assertEqual(900, sampler()) + + def testManagerAndMeter(self): + manager = counters._CounterManager() + total = counters._TotalCounter('test.mytotal', 'Example total counter') + delta = counters._DeltaCounter('test.mydelta', 'Example delta counter') + manager.register(total) + manager.register(delta) + + # Test that counters are accessible from manager object via dot notation. + self.assertIs(manager.test.mytotal, total) + self.assertIs(manager.test.mydelta, delta) + + # Test for namespace errors when registering a counter with the manager. + badcounter = counters._RateCounter('test', 'Bad namespace') + with self.assertRaises(KeyError): + manager.register(badcounter) + + badcounter = counters._RateCounter('test.mydelta.rate', 'Existing counter') + with self.assertRaises(KeyError): + manager.register(badcounter) + + # Test basic functionality of meter class in conjunction with a meter. + meter = counters.Meter(manager.test) + + total.increment() + total.increment(5) + delta.increment(6) + + x = meter.sample() + self.assertEqual(6, x.test.mytotal) + self.assertEqual(6, x.test.mydelta) + + total.increment(5) + delta.increment(6) + x = meter.sample() + self.assertEqual(11, x.test.mytotal) + self.assertEqual(6, x.test.mydelta) + + # Test that namespace selection using a meter has the appropriate behavior. + + d = meter.describe() + self.assertEqual(d.test.mytotal, 'Example total counter') + diff --git a/backend/base/test/handler_test.py b/backend/base/test/handler_test.py new file mode 100644 index 0000000..49cb129 --- /dev/null +++ b/backend/base/test/handler_test.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder web request handler tests. + + HandlerTestCase: sets up an HTTP server-based test case with + ViewfinderHandler-derived handlers using asynchronous decorators. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import random +import struct +import sys +import unittest + +from functools import partial +from tornado import httpclient, options, web, testing +from tornado.stack_context import StackContext + + +from viewfinder.backend.base import handler, util +from viewfinder.backend.base.testing import async_test, BaseTestCase +from viewfinder.backend.db.local_client import LocalClient +from viewfinder.backend.db.db_client import DBClient, DBKey, DBKeySchema + + +class _DatastoreHandler(web.RequestHandler): + """Handler for datastore database set/retrieval.""" + @handler.asynchronous(datastore=True) + def get(self): + def _OnGet(result): + self.write('%s' % result.attributes['v']) + self.finish() + + self._client.GetItem(table='test', + key=DBKey(hash_key=self.get_argument('k'), range_key=None), + attributes=['v'], callback=_OnGet) + + @handler.asynchronous(datastore=True) + def post(self): + def _OnPut(result): + self.write('ok') + self.finish() + + self._client.PutItem(table='test', + key=DBKey(hash_key=self.get_argument('k'), range_key=None), + attributes={'v': self.get_argument('v')}, callback=_OnPut) + + +class HandlerTestCase(BaseTestCase, testing.AsyncHTTPTestCase): + """Sets up a web server which handles various backend asynchronous + services, such as datastore db operations. + """ + def setUp(self): + super(HandlerTestCase, self).setUp() + + # Redefine http client to increase the maximum number of outstanding clients. + self.http_client = httpclient.AsyncHTTPClient( + io_loop=self.io_loop, max_clients=100, force_instance=True) + # Setup a test table in a test datastore client instance. + options.options.localdb_dir = '' + + DBClient.SetInstance(LocalClient(None)) + DBClient.Instance().CreateTable( + table='test', hash_key_schema=DBKeySchema(name='k', value_type='S'), + range_key_schema=None, read_units=10, write_units=5, callback=None) + + def get_app(self): + """Creates a web server which handles: + + - GET /datastore?k= - retrieve value for ; shard is hash of key + - POST /datastore?k=&v= - set datastore :; shard is hash of key + """ + return web.Application([(r"/datastore", _DatastoreHandler)]) + + @async_test + def testDatastore(self): + """Test the webserver handles datastore key/value store and retrieval + by inserting a collection of random values and verifying their + retrieval, in parallel. + """ + values = self._CreateRandomValues(num_values=100) + + def _InsertDone(): + self._RetrieveValues(values, self.stop) + + self._InsertValues(values, _InsertDone) + + def _CreateRandomValues(self, num_values=100): + """Creates num_values random integers between [0, 1<<20) + """ + return [int(random.uniform(0, 1 << 20)) for i in xrange(num_values)] + + def _InsertValues(self, values, callback): + """Inserts values into datastore via the tornado web server and + invokes callback with the sequence of values when complete. The + values are randomly distributed over the available shards. + + - The key of each value is computed as: 'k%d' % value + """ + def _VerifyResponse(cb, resp): + self.assertEqual(resp.body, 'ok') + cb() + + with util.Barrier(callback) as b: + for val in values: + self.http_client.fetch( + httpclient.HTTPRequest(self.get_url('/datastore'), method='POST', + body='k=k%d&v=%d' % (val, val)), + callback=partial(_VerifyResponse, b.Callback())) + + def _RetrieveValues(self, values, callback): + """Retrieves and verifies the specified values from Datastore database + via the tornado web server. + """ + def _VerifyResponse(val, cb, resp): + self.assertEqual(resp.body, repr(val)) + cb() + + with util.Barrier(callback) as b: + for val in values: + self.http_client.fetch( + httpclient.HTTPRequest(self.get_url('/datastore?k=k%d' % val), method='GET'), + callback=partial(_VerifyResponse, val, b.Callback())) + diff --git a/backend/base/test/message_test.py b/backend/base/test/message_test.py new file mode 100644 index 0000000..13bf90b --- /dev/null +++ b/backend/base/test/message_test.py @@ -0,0 +1,276 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Test message validation and migration. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +from copy import deepcopy +from tornado.ioloop import IOLoop +from viewfinder.backend.base import testing +from viewfinder.backend.base.message import Message, MessageMigrator, BadMessageException, REQUIRED_MIGRATORS + +class RenameTestMigrator(MessageMigrator): + """Rename a field in the message.""" + def __init__(self): + MessageMigrator.__init__(self, Message.TEST_VERSION) + + def MigrateForward(self, client, message, callback): + assert message.dict['headers'] + message.dict['renamed-scalar'] = message.dict['scalar'] + del message.dict['scalar'] + callback() + + def MigrateBackward(self, client, message, callback): + assert message.dict['headers'] + message.dict['scalar'] = message.dict['renamed-scalar'] + del message.dict['renamed-scalar'] + IOLoop.current().add_callback(callback) + + +class MessageTestCase(testing.BaseTestCase): + SCHEMA_NO_VERSION = { + 'description': 'test schema', + 'type': 'object', + 'properties': { + 'scalar': {'type': 'string', 'blank': True}, + 'list': { + 'type': 'array', + 'items': {'type': 'any'}, + }, + 'sub-dict': { + 'description': 'nested dictionary', + 'required': False, + 'type': 'object', + 'properties': { + 'none': {'type': 'null'}, + 'sub-scalar': {'type': 'string'}, + 'sub-list': { + 'type': 'array', + 'items': { + 'description': 'dictionary in list', + 'type': 'object', + 'properties': { + 'value': {'type': 'number'} + }, + }, + }, + }, + }, + }, + } + + SCHEMA_WITH_VERSION = deepcopy(SCHEMA_NO_VERSION) + SCHEMA_WITH_VERSION['properties']['headers'] = { + 'description': 'defines required version field', + 'required': False, + 'type': 'object', + 'properties': { + 'version': {'type': 'number', 'minimum': Message.ADD_HEADERS_VERSION}, + }, + } + + RENAMED_SCHEMA_WITH_VERSION = deepcopy(SCHEMA_WITH_VERSION) + del RENAMED_SCHEMA_WITH_VERSION['properties']['scalar'] + RENAMED_SCHEMA_WITH_VERSION['properties']['renamed-scalar'] = {'type': 'string'} + + MSG_NO_VERSION = { + 'scalar': 'Simple scalar field', + 'list': [10, 'list item', []], + 'sub-dict': { + 'none': None, + 'sub-scalar': 'Simple scalar field within sub-dict', + 'sub-list': [{'value': 20, 'extra': 'extra field that does not appear in schema'}], + 'extra': 'extra field that does not appear in schema', + }, + 'extra': 'extra field that does not appear in schema', + } + + MSG_WITH_VERSION = deepcopy(MSG_NO_VERSION) + MSG_WITH_VERSION['headers'] = dict(version=Message.ADD_HEADERS_VERSION) + + def testMessage(self): + """Basic tests of the Message class.""" + # Message with no version, no schema. + self._TestMessage(MessageTestCase.MSG_NO_VERSION, original_version=Message.INITIAL_VERSION) + + # Message with version, no schema. + self._TestMessage(MessageTestCase.MSG_WITH_VERSION, original_version=Message.ADD_HEADERS_VERSION) + + # Default version. + self._TestMessage(MessageTestCase.MSG_NO_VERSION, default_version=Message.ADD_HEADERS_VERSION, + original_version=Message.ADD_HEADERS_VERSION) + message_dict = deepcopy(MessageTestCase.MSG_NO_VERSION) + message_dict['headers'] = {} + self._TestMessage(message_dict, default_version=Message.ADD_HEADERS_VERSION, + original_version=Message.ADD_HEADERS_VERSION) + + # ERROR: Message version not present. + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + del message_dict['headers']['version'] + self.assertRaises(BadMessageException, self._TestMessage, message_dict) + + # ERROR: Message version was present, but anachronistic (i.e. try to use headers in version + # that didn't support them). + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + message_dict['headers']['version'] = Message.INITIAL_VERSION + self.assertRaises(BadMessageException, self._TestMessage, message_dict) + + # ERROR: Message version not high enough to be supported. + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + message_dict['headers']['version'] = -1 + self.assertRaises(BadMessageException, self._TestMessage, message_dict) + + self.assertRaises(BadMessageException, self._TestMessage, MessageTestCase.MSG_WITH_VERSION, + min_supported_version=Message.TEST_VERSION) + + # ERROR: Message version not low enough to be supported. + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + message_dict['headers']['version'] = Message.ADD_HEADERS_VERSION + self.assertRaises(BadMessageException, self._TestMessage, message_dict, + max_supported_version=Message.INITIAL_VERSION) + + # Min required version specified in the message. + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + message_dict['headers']['version'] = 1000 + message_dict['headers']['min_required_version'] = Message.ADD_HEADERS_VERSION + self._TestMessage(message_dict, original_version=Message.MAX_VERSION) + + message_dict['headers']['version'] = Message.ADD_HEADERS_VERSION + self._TestMessage(message_dict, original_version=Message.ADD_HEADERS_VERSION) + + # ERROR: Min required version specified in the message, but greater than max supported version. + message_dict['headers']['version'] = 3 + message_dict['headers']['min_required_version'] = Message.TEST_VERSION + self.assertRaises(BadMessageException, self._TestMessage, message_dict, + max_supported_version=Message.ADD_HEADERS_VERSION) + + # ERROR: Min required version specified in the message, but less than min required message version. + message_dict['headers']['version'] = Message.TEST_VERSION + message_dict['headers']['min_required_version'] = Message.ADD_HEADERS_VERSION + self.assertRaises(BadMessageException, self._TestMessage, message_dict, + min_supported_version=1000, + max_supported_version=Message.ADD_HEADERS_VERSION) + + # ERROR: Min required version specified in the message, but greater than version. + message_dict = deepcopy(MessageTestCase.MSG_WITH_VERSION) + message_dict['headers']['min_required_version'] = 100 + self.assertRaises(BadMessageException, self._TestMessage, message_dict) + + # Message with sanitize + schema. + self._TestMessage(MessageTestCase.MSG_NO_VERSION, original_version=Message.INITIAL_VERSION, + schema=MessageTestCase.SCHEMA_NO_VERSION, sanitize=True) + + # Message with allow_extra_fields=True. + self._TestMessage(MessageTestCase.MSG_NO_VERSION, original_version=Message.INITIAL_VERSION, + schema=MessageTestCase.SCHEMA_NO_VERSION, allow_extra_fields=True) + + # ERROR: Message violates schema due to extra fields. + self.assertRaises(BadMessageException, self._TestMessage, MessageTestCase.MSG_NO_VERSION, + original_version=Message.INITIAL_VERSION, schema=MessageTestCase.SCHEMA_NO_VERSION) + + # Visit message. + def _TestVisitor(key, value): + """Remove extra fields, replace "list" field.""" + if key == 'extra': + return () + elif key == 'list': + return ('new-field', 'new value') + + message = Message(deepcopy(MessageTestCase.MSG_NO_VERSION)) + message.Visit(_TestVisitor) + message.Visit(self._TestExtraField) + self.assertTrue(message.dict.has_key('scalar')) + self.assertTrue(message.dict.has_key('new-field')) + self.assertFalse(message.dict.has_key('list')) + + def testMigrate(self): + """Test version migration functionality on the Message class.""" + # Migrate message with no header to have a header. + message = self._TestMessage(MessageTestCase.MSG_NO_VERSION, + original_version=Message.INITIAL_VERSION, + max_supported_version=Message.INITIAL_VERSION, + schema=MessageTestCase.SCHEMA_WITH_VERSION, + allow_extra_fields=True, + migrate_version=Message.ADD_HEADERS_VERSION) + + # Migrate message with header to have no header. + message = self._TestMessage(message.dict, + sanitize=True, + schema=MessageTestCase.SCHEMA_NO_VERSION, + original_version=Message.ADD_HEADERS_VERSION, + migrate_version=Message.INITIAL_VERSION) + + # Add a migrator to the list of migrators and migrate from initial version. + message = self._TestMessage(MessageTestCase.MSG_NO_VERSION, + original_version=Message.INITIAL_VERSION, + sanitize=True, + schema=MessageTestCase.RENAMED_SCHEMA_WITH_VERSION, + migrate_version=Message.TEST_VERSION, + migrators=[RenameTestMigrator()]) + renamed_dict = message.dict + + # Migrate message with renamed field all the way back to initial version. + message = self._TestMessage(renamed_dict, + original_version=Message.TEST_VERSION, + schema=MessageTestCase.SCHEMA_NO_VERSION, + migrate_version=Message.INITIAL_VERSION, + migrators=[RenameTestMigrator()]) + + # Migrate message with renamed field back to add headers version (not all the way). + message = self._TestMessage(renamed_dict, + original_version=Message.TEST_VERSION, + schema=MessageTestCase.SCHEMA_WITH_VERSION, + migrate_version=Message.ADD_HEADERS_VERSION, + migrators=[RenameTestMigrator()]) + + # Migrate message with renamed field back to initial version. + message = self._TestMessage(message.dict, + original_version=Message.ADD_HEADERS_VERSION, + schema=MessageTestCase.SCHEMA_NO_VERSION, + migrate_version=Message.INITIAL_VERSION, + migrators=[RenameTestMigrator()]) + + # No migration necessary. + message = self._TestMessage(message.dict, + original_version=Message.INITIAL_VERSION, + schema=MessageTestCase.SCHEMA_NO_VERSION, + migrate_version=Message.INITIAL_VERSION, + migrators=[RenameTestMigrator()]) + + def _TestMessage(self, message_dict, original_version=None, default_version=Message.INITIAL_VERSION, + min_supported_version=Message.INITIAL_VERSION, max_supported_version=Message.MAX_VERSION, + schema=None, allow_extra_fields=False, sanitize=False, migrate_version=None, migrators=None): + # Create the message. + message_dict = deepcopy(message_dict) + message = Message(message_dict, default_version=default_version, min_supported_version=min_supported_version, + max_supported_version=max_supported_version) + self.assertEqual(message.dict, message_dict) + self.assertEqual(message.original_version, original_version) + self.assertEqual(message.version, original_version) + self.assertTrue(message.version == Message.INITIAL_VERSION or message.dict['headers'].has_key('version')) + + # Migrate the message to "migrate_version". + if migrate_version is not None: + migrators = None if migrators is None else sorted(REQUIRED_MIGRATORS + migrators) + message.Migrate(None, migrate_version, lambda message: self.stop(message), migrators) + self.assert_(self.wait() is message) + self.assertEqual(message.version, migrate_version) + + # Sanitize the message if requested. + if sanitize: + message.Validate(schema, True) + message.Sanitize() + message.Visit(self._TestExtraField) + + # Validate the message according to "schema". + if schema: + message.Validate(schema, allow_extra_fields) + self.assertEqual(message.schema, schema) + if not allow_extra_fields: + message.Visit(self._TestExtraField) + + return message + + def _TestExtraField(self, key, value): + self.assertNotEqual(key, 'extra') diff --git a/backend/base/test/retry_test.py b/backend/base/test/retry_test.py new file mode 100644 index 0000000..da1fcfa --- /dev/null +++ b/backend/base/test/retry_test.py @@ -0,0 +1,151 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Retry module tests.""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import functools +import logging +import contextlib +from datetime import timedelta +from tornado import stack_context, testing +from viewfinder.backend.base import retry, util +from viewfinder.backend.base.testing import BaseTestCase, LogMatchTestCase + +class CallWithRetryTestCase(BaseTestCase, LogMatchTestCase): + + def testWithStackContext1(self): + """Ensure Retry preserves StackContext.""" + self.__in_context = False + + @contextlib.contextmanager + def _MyContext(): + try: + self.__in_context = True + yield + finally: + self.__in_context = False + + def _OnCompletedCheckContext(result, error): + self.assertTrue(self.__in_context) + self.stop() + + with stack_context.StackContext(_MyContext): + retry_policy = retry.RetryPolicy(max_tries=2, check_result=lambda res, err: err) + retry.CallWithRetryAsync(retry_policy, self._AsyncFuncFailOnce, callback=_OnCompletedCheckContext) + self.wait() + + def testWithStackContext2(self): + """Ensure Retry doesn't interfere with asynchronous function that throws immediately.""" + try: + with stack_context.ExceptionStackContext(self._OnError): + retry.CallWithRetryAsync(retry.RetryPolicy(), self._AsyncFuncRaisesError, + callback=self._OnCompleted) + self.assert_(False, 'Expected exception to be raised') + except: + self.wait() + + def testWithStackContext3(self): + """Ensure Retry doesn't interfere with asynchronous callback that throws.""" + try: + with stack_context.ExceptionStackContext(self._OnError): + retry.CallWithRetryAsync(retry.RetryPolicy(check_exception=lambda typ, val, tb: True), self._AsyncFunc, + callback=self._OnCompletedRaisesError) + self.wait() + self.assert_(False, 'Expected exception to be raised') + except Exception as e: + self.assert_('_OnCompletedRaisesError' in e.message, e) + + def testWithBarrier(self): + """Ensure Retry doesn't interfere with barriers.""" + retry_policy = retry.RetryPolicy(max_tries=2, check_result=lambda res, err: err) + with util.MonoBarrier(self._OnCompleted) as b: + retry.CallWithRetryAsync(retry_policy, self._AsyncFuncFailOnce, callback=b.Callback()) + self.wait() + + def testRetryPolicyApi(self): + """Test RetryPolicy __init__ API.""" + self.assertRaises(OverflowError, functools.partial(retry.RetryPolicy, timeout=1234123412341234)) + + retry.RetryPolicy(timeout=timedelta(milliseconds=500)) + self.assertEqual(retry.RetryPolicy(timeout=10).timeout.total_seconds(), 10) + self.assertEqual(retry.RetryPolicy(timeout= -1.5).timeout.total_seconds(), -1.5) + + retry.RetryPolicy(min_delay=timedelta(days=500)) + self.assertEqual(retry.RetryPolicy(min_delay=10).min_delay.total_seconds(), 10) + self.assertEqual(retry.RetryPolicy(min_delay= -1.5).min_delay.total_seconds(), -1.5) + + retry.RetryPolicy(max_delay=timedelta(hours=500)) + self.assertEqual(retry.RetryPolicy(max_delay=10).max_delay.total_seconds(), 10) + self.assertEqual(retry.RetryPolicy(max_delay= -1.5).max_delay.total_seconds(), -1.5) + + def testMaxTries(self): + """Test retry scenario in which the RetryPolicy max_tries is exceeded.""" + retry_policy = retry.RetryPolicy(max_tries=10, check_result=lambda res, err: True) + retry.CallWithRetryAsync(retry_policy, self._AsyncFunc, callback=self._OnCompleted) + self.wait() + self.assertLogMatches('Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying', + 'Expected 9 retries in the log') + + def testTimeoutAndDelays(self): + """Test retry scenario in which the RetryPolicy timeout is exceeded.""" + retry_policy = retry.RetryPolicy(timeout=.6, min_delay=.05, max_delay=.2, check_result=lambda res, err: True) + retry.CallWithRetryAsync(retry_policy, self._AsyncFunc, callback=self._OnCompleted) + self.wait() + self.assertLogMatches('Retrying.*Retrying.*Retrying', + 'Expected at least 3 retries in the log') + + def testCallWithRetryApi(self): + """Test CallWithRetry API.""" + self.assertRaises(AssertionError, retry.CallWithRetryAsync, None, None) + + def testRetryWithException(self): + """Retry on exceptions raised immediately by async function.""" + def CallWithRetry(): + retry_policy = retry.RetryPolicy(max_tries=3, check_exception=lambda typ, val, tb: True) + retry.CallWithRetryAsync(retry_policy, self._AsyncFuncRaisesErrorOnce, dict(), callback=self.stop) + + self.io_loop.add_callback(CallWithRetry) + self.wait() + + def testRetryWithException2(self): + """Retry on exceptions raised by async function after stack transfer.""" + def CallAfterStackTransfer(dict, callback): + func = functools.partial(self._AsyncFuncRaisesErrorOnce, dict, callback) + self.io_loop.add_callback(func) + + retry_policy = retry.RetryPolicy(max_tries=3, check_exception=lambda typ, val, tb: True) + retry.CallWithRetryAsync(retry_policy, CallAfterStackTransfer, dict(), callback=self.stop) + self.wait() + + def _AsyncFuncRaisesErrorOnce(self, dict, callback): + if not 'raised_error' in dict: + dict['raised_error'] = True + raise Exception('Error in AsyncFuncRaisesErrorOnce') + callback() + + def _AsyncFunc(self, callback=None): + func = functools.partial(callback, 'hello world', None) + self.io_loop.add_callback(func) + + def _AsyncFuncRaisesError(self, callback=None): + raise Exception('Error in _AsyncFuncRaisesError') + + def _AsyncFuncFailOnce(self, callback=None): + if self.__dict__.has_key('_succeed_async_func_fail_once'): + func = functools.partial(callback, 'hello world', None) + del self._succeed_async_func_fail_once + else: + func = functools.partial(callback, None, Exception('Failed')) + self._succeed_async_func_fail_once = True + + self.io_loop.add_callback(func) + + def _OnCompleted(self, result, error): + self.stop() + + def _OnCompletedRaisesError(self, result, error): + raise Exception('Error in _OnCompletedRaisesError') + + def _OnError(self, exc_type, value, traceback): + self.stop() diff --git a/backend/base/test/secrets_test.py b/backend/base/test/secrets_test.py new file mode 100755 index 0000000..2ab5190 --- /dev/null +++ b/backend/base/test/secrets_test.py @@ -0,0 +1,243 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Secrets test. + + Test secrets module. user vs shared, encrypted vs plain. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import getpass +import json +import logging +import mock +import os +import shutil +import tempfile +import unittest + +from tornado import options +from viewfinder.backend.base import ami_metadata, base_options, secrets, testing +from viewfinder.backend.base.exceptions import CannotReadEncryptedSecretError + +class SecretsTestCase(unittest.TestCase): + def setUp(self): + # Fake out the keyring to None for the entire test. + self._prev_keyring = secrets.keyring + secrets.keyring = None + + self._domain = options.options.domain + self._prev_user_dir = options.options.user_secrets_dir + self._prev_shared_dir = options.options.secrets_dir + self._prev_devbox = options.options.devbox + + # Create tmp directories and set flag values. + self._user_dir = tempfile.mkdtemp() + options.options.user_secrets_dir = self._user_dir + os.mkdir(os.path.join(self._user_dir, self._domain)) + + self._shared_dir = tempfile.mkdtemp() + options.options.secrets_dir = self._shared_dir + os.mkdir(os.path.join(self._shared_dir, self._domain)) + + + def tearDown(self): + # Recursively delete temp directories and restore flag values. + shutil.rmtree(self._user_dir) + shutil.rmtree(self._shared_dir) + + options.options.user_secrets_dir = self._prev_user_dir + options.options.secrets_dir = self._prev_shared_dir + options.options.devbox = self._prev_devbox + + secrets.keyring = self._prev_keyring + secrets._user_secrets_manager = None + secrets._shared_secrets_manager = None + + def testNoDomainDir(self): + """Test secrets manager without a domain dir.""" + mgr = secrets.SecretsManager('test', 'fake_domain', self._shared_dir) + # We do not fail on Init since we want to be able to support non-existent user secrets. + mgr.Init() + + # Behaves just like an empty secrets manager. + self.assertEqual(len(mgr.ListSecrets()), 0) + + # Trying to add a secret fails. + self.assertRaises(IOError, mgr.PutSecret, 'foo', 'codeforfoo') + + + def testPlain(self): + """Test secrets manager with plain-text secrets.""" + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + # Empty directory, Init will not require a passphrase. + mgr.Init() + + self.assertEqual(len(mgr.ListSecrets()), 0) + self.assertRaises(KeyError, mgr.GetSecret, 'foo') + self.assertFalse(mgr.HasSecret('foo')) + + # Put a secret, but underlying directory doesn't exist (switch domains first). + mgr.PutSecret('foo', 'codeforfoo') + self.assertTrue(mgr.HasSecret('foo')) + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + self.assertEqual(len(mgr.ListSecrets()), 1) + + # Now check that the underlying file exists. + with open(os.path.join(self._shared_dir, self._domain, 'foo')) as f: + self.assertEqual(f.read(), 'codeforfoo') + + # Overwrite secret. + mgr.PutSecret('foo', 'newcodeforfoo') + self.assertEqual(mgr.GetSecret('foo'), 'newcodeforfoo') + self.assertEqual(len(mgr.ListSecrets()), 1) + + # Now check that the underlying file exists. + with open(os.path.join(self._shared_dir, self._domain, 'foo')) as f: + self.assertEqual(f.read(), 'newcodeforfoo') + + # Create a new secrets manager. + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertTrue(mgr.HasSecret('foo')) + self.assertEqual(mgr.GetSecret('foo'), 'newcodeforfoo') + self.assertEqual(len(mgr.ListSecrets()), 1) + + # Passing a passphrase as a flag does not impact plain-text secrets. + options.options.passphrase = 'not a passphrase' + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertEqual(mgr.GetSecret('foo'), 'newcodeforfoo') + + + def testEncrypted(self): + """Test secrets manager with encrypted secrets.""" + + # The only way to make a secret manager encrypt when empty is to ask it + # to prompt for a passphrase. It does so using getpass.getpass. + passphrase = 'my voice is my passport!' + with mock.patch.object(secrets.getpass, 'getpass') as getpass: + getpass.return_value = passphrase + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init(should_prompt=True) + + # Secret will be encrypted. + mgr.PutSecret('foo', 'codeforfoo') + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + with open(os.path.join(self._shared_dir, self._domain, 'foo')) as f: + contents = f.read() + self.assertNotEqual(contents, 'codeforfoo') + (cipher, ciphertext) = json.loads(contents) + self.assertEqual(cipher, 'AES') + # TODO(marc): maybe we should test the encryption itself. + + # Now create a new secrets manager. We do not ask it to prompt, it will figure it out + # all by itself. It does this in a number of ways: + + + ##################### --devbox=False ######################## + options.options.devbox = False + + # Set stdin to raise an exception, just to make sure we're not using it. + with mock.patch.object(secrets.getpass, 'getpass') as getpass: + getpass.side_effect = Exception('you should not be using stdin in --devbox=False mode') + # Uses --passphrase if specified. + options.options.passphrase = passphrase + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + + # We get an assertion error when a passphrase is supplied but bad. This is because it fails on sha sum. + options.options.passphrase = 'bad passphrase' + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + self.assertRaises(AssertionError, mgr.Init) + + # Uses AMI metadata otherwise. + options.options.passphrase = None + # No AMI fetched, or passphrase not one of the fetched fields. + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + self.assertRaisesRegexp(CannotReadEncryptedSecretError, 'failed to fetch passphrase from AWS instance metadata', + mgr.Init) + + # Good passphrase from AMI metadata. + ami_metadata.SetAMIMetadata({'user-data/passphrase': passphrase}) + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + + # Bad passphrase from AMI metadata. + ami_metadata.SetAMIMetadata({'user-data/passphrase': 'not a good passphrase.'}) + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + self.assertRaises(AssertionError, mgr.Init) + + + ##################### --devbox=True ######################## + options.options.devbox = True + # Set bad AMI metadata just to show that we never use it. + ami_metadata.SetAMIMetadata({'user-data/passphrase': 'not a good passphrase.'}) + + # Uses --passphrase if specified. + options.options.passphrase = passphrase + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + + # If --passphrase is None and we cannot prompt, we have no way of getting the passphrase. + options.options.passphrase = None + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + self.assertRaisesRegexp(CannotReadEncryptedSecretError, 'passphrase is required but was not provided', + mgr.Init, can_prompt=False) + + # Passphrase is read from stdin if prompting is allowed. + with mock.patch.object(secrets.getpass, 'getpass') as getpass: + getpass.return_value = passphrase + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + mgr.Init() + self.assertEqual(mgr.GetSecret('foo'), 'codeforfoo') + + # Pass a bad passphrase on stdin. + with mock.patch.object(secrets.getpass, 'getpass') as getpass: + getpass.return_value = 'not a good passphrase' + mgr = secrets.SecretsManager('test', self._domain, self._shared_dir) + self.assertRaises(AssertionError, mgr.Init) + + + def testMultipleManagers(self): + """Test the secrets managers in their natural habitat: automatic selection of user vs shared based on flags.""" + # these may not be None if we've been running other tests using run-tests. + secrets._user_secrets_manager = None + secrets._shared_secrets_manager = None + + # Devbox mode: init user secrets, and lazily init shared secrets is requesting a secret on in user secrets. + options.options.devbox = True + secrets.InitSecrets() + self.assertIsNotNone(secrets._user_secrets_manager) + self.assertIsNone(secrets._shared_secrets_manager) + + # Request a secret contained in user secrets: shared secrets remain uninitialized. + secrets._user_secrets_manager.PutSecret('foo', 'codeforfoo') + self.assertEqual(secrets.GetSecret('foo'), 'codeforfoo') + self.assertIsNotNone(secrets._user_secrets_manager) + self.assertIsNone(secrets._shared_secrets_manager) + + # Request a secret not contained anywhere. As soon as we notice that it's not in user secrets, we initialize + # the shared secrets and look there, which fails. + self.assertRaises(KeyError, secrets.GetSecret, 'bar') + self.assertIsNotNone(secrets._user_secrets_manager) + self.assertIsNotNone(secrets._shared_secrets_manager) + + # Non-devbox mode: user secrets are never used. shared secrets are initialized right away. + options.options.devbox = False + secrets._user_secrets_manager = None + secrets._shared_secrets_manager = None + + secrets.InitSecrets() + self.assertIsNone(secrets._user_secrets_manager) + self.assertIsNotNone(secrets._shared_secrets_manager) + + # Lookup whatever we want, we still won't use the user secrets.:w + secrets._shared_secrets_manager.PutSecret('foo', 'codeforfoo') + self.assertEqual(secrets.GetSecret('foo'), 'codeforfoo') + self.assertRaises(KeyError, secrets.GetSecret, 'bar') + self.assertIsNone(secrets._user_secrets_manager) + self.assertIsNotNone(secrets._shared_secrets_manager) diff --git a/backend/base/test/statistics_test.py b/backend/base/test/statistics_test.py new file mode 100755 index 0000000..81e63bc --- /dev/null +++ b/backend/base/test/statistics_test.py @@ -0,0 +1,74 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Statistics tests. +""" + +__authors__ = ['marc@emailscrubbed.com (Marc Berhault)'] + +import logging +try: + import numpy +except ImportError: + numpy = None +import random +import unittest + +from viewfinder.backend.base import statistics, testing + +@unittest.skipIf(numpy is None, 'numpy not present') +class FormatStatsTestCase(testing.BaseTestCase): + def _RandomList(self): + """ List of random integers (0-20) and of random size (10-20). """ + l = [] + for i in range(random.randint(10, 20)): + l.append(random.randint(0, 20)) + print 'list: %r' % l + return l + + def testEmpty(self): + self.assertEqual(statistics.FormatStats([]), '') + + def testSimple(self): + """ No indentation, no percentiles. """ + a = self._RandomList() + out_str = 'mean=%.2f\nmedian=%.2f\nstddev=%.2f' % \ + (numpy.mean(a), numpy.median(a), numpy.std(a)) + self.assertEqual(statistics.FormatStats(a), out_str) + + def testIndent(self): + """ With indentation, no percentiles. """ + a = self._RandomList() + out_str = ' mean=%.2f\n median=%.2f\n stddev=%.2f' % \ + (numpy.mean(a), numpy.median(a), numpy.std(a)) + self.assertEqual(statistics.FormatStats(a, indent=3), out_str) + + def testPercentile(self): + """ With indentation and percentiles. """ + a = self._RandomList() + p = [80, 90, 95, 99] + out_str = ' mean=%.2f\n median=%.2f\n stddev=%.2f\n' % \ + (numpy.mean(a), numpy.median(a), numpy.std(a)) + out_str += ' 80/90/95/99 percentiles=%s' % numpy.percentile(a, p) + self.assertEqual(statistics.FormatStats(a, percentiles=p, indent=3), out_str) + +@unittest.skipIf(numpy is None, 'numpy not present') +class HistogramTestCase(testing.BaseTestCase): + def testEmpty(self): + self.assertEqual(statistics.HistogramToASCII([]), '') + + def testSimple(self): + a = [1, 1, 2, 4, 4] + out_str = ' [1-2) 2 40.00% ########\n' + out_str += ' [2-3) 1 20.00% ####\n' + out_str += ' [3-4] 2 40.00% ########' + self.assertEqual(statistics.HistogramToASCII(a, bins=3, indent=2, line_width=26), out_str) + + def testRounding(self): + """ Bucket limits can be floats, but we round everything for display. """ + a = [1, 1, 2, 4, 4] + out_str = ' [1-1) 2 40.00% ########\n' + out_str += ' [1-2) 1 20.00% ####\n' + out_str += ' [2-2) 0 0.00% \n' + out_str += ' [2-3) 0 0.00% \n' + out_str += ' [3-4] 2 40.00% ########' + self.assertEqual(statistics.HistogramToASCII(a, bins=5, indent=2, line_width=26), out_str) diff --git a/backend/base/test/testing_test.py b/backend/base/test/testing_test.py new file mode 100644 index 0000000..8b1164a --- /dev/null +++ b/backend/base/test/testing_test.py @@ -0,0 +1,53 @@ +import mock + +from cStringIO import StringIO +from tornado import httpclient +from viewfinder.backend.base import testing + +kURL = "http://www.example.com/" + +class MockAsyncHTTPClientTestCase(testing.BaseTestCase): + def setUp(self): + super(MockAsyncHTTPClientTestCase, self).setUp() + self.http_client = testing.MockAsyncHTTPClient() + + def test_unmapped(self): + """Requests not on the whitelist raise an error.""" + with self.assertRaises(ValueError): + self.http_client.fetch(kURL, self.stop) + + def test_string(self): + """Map a url to a constant string.""" + self.http_client.map(kURL, "hello world") + self.http_client.fetch(kURL, self.stop) + response = self.wait() + self.assertEqual(response.body, "hello world") + + def test_callable(self): + """Map a url to a function returning a string.""" + self.http_client.map(kURL, lambda request: "hello world") + self.http_client.fetch(kURL, self.stop) + response = self.wait() + self.assertEqual(response.body, "hello world") + + def test_response(self): + """Map a url to a function returning an HTTPResponse. + + HTTPResponse's constructor requires a request object, so there is no + fourth variant that returns a constant HTTPResponse. + """ + self.http_client.map(kURL, lambda request: httpclient.HTTPResponse( + request, 404, buffer=StringIO(""))) + self.http_client.fetch(kURL, self.stop) + response = self.wait() + self.assertEqual(response.code, 404) + + def test_with_patch(self): + """Replace the AsyncHTTPClient class using mock.patch.""" + self.http_client.map(kURL, "hello world") + with mock.patch('tornado.httpclient.AsyncHTTPClient', self.http_client): + real_client = httpclient.AsyncHTTPClient() + self.assertIs(self.http_client, real_client) + real_client.fetch(kURL, self.stop) + response = self.wait() + self.assertEqual(response.body, "hello world") diff --git a/backend/base/test/util_test.py b/backend/base/test/util_test.py new file mode 100755 index 0000000..960b0a9 --- /dev/null +++ b/backend/base/test/util_test.py @@ -0,0 +1,478 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Utility tests. + + ParseHostPort(): parses host:port string and returns tuple +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)' + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging +import time +import unittest +from datetime import date, timedelta +from functools import partial + +from viewfinder.backend.base import util, testing + +class BarrierTestCase(testing.BaseTestCase): + """Tests for basic barrier type.""" + def testBarrier(self): + val = [False] + def _Callback(): + val[0] = True + self.stop() + + with util.Barrier(_Callback) as b: + cb1 = b.Callback() + cb2 = b.Callback() + cb1() + self.io_loop.add_callback(self.stop) + self.wait() + self.assertFalse(val[0]) + cb2() + self.wait() + self.assertTrue(val[0]) + + def testEmptyBarrier(self): + val = [False] + def _Callback(): + val[0] = True + self.stop() + + with util.Barrier(_Callback): + pass + + self.wait() + self.assertTrue(val[0]) + + def testEmptyBarrierException(self): + val = [False] + def _Exception(type_, value_, traceback): + print "Exception" + self.io_loop.add_callback(self.stop) + + def _Completed(): + print "Completed" + val[0] = True + + with util.Barrier(_Completed, _Exception): + raise KeyError('Key') + + self.wait() + self.assertFalse(val[0], 'Barrier complete method was called unexpectedly.') + + def testCompletedBeforeException(self): + """Make the barrier callback and then raise exception.""" + val = [0] + def _Exception(type_, value_, traceback): + logging.info("Exception") + val[0] += 1 + + def _Completed(): + logging.info("Completed") + val[0] += 1 + + def _RaiseException(): + raise KeyError('key') + + def _PropException(type_, value_, traceback): + self.io_loop.add_callback(self.stop) + + with util.ExceptionBarrier(_PropException): + with util.Barrier(_Completed, _Exception): + self.io_loop.add_callback(_RaiseException) + + self.wait() + self.assertEqual(val[0], 1, 'Both _Completed and _Exception were called.') + + def testCompletedAfterException(self): + """Raise exception and then make the barrier callback.""" + val = [0] + def _Exception(type_, value_, traceback): + logging.info("Exception") + val[0] += 1 + self.io_loop.add_callback(self.stop) + + def _Completed(): + logging.info("Completed") + val[0] += 1 + self.io_loop.add_callback(self.stop) + + def _RaiseException(completed_cb): + self.io_loop.add_callback(partial(completed_cb, 1)) + raise KeyError('key') + + with util.ArrayBarrier(_Completed, on_exception=_Exception) as b: + self.io_loop.add_callback(partial(_RaiseException, b.Callback())) + self.io_loop.add_callback(partial(_RaiseException, b.Callback())) + + self.wait() + self.assertEqual(val[0], 1, 'Both _Completed and _Exception were called.') + + +class MonoBarrierTestCase(testing.BaseTestCase): + """Tests for MonoBarrier barrier type.""" + def testBarrier(self): + val = [] + def _Callback(result): + val.append(result) + self.stop() + + with util.MonoBarrier(_Callback) as b: + cb = b.Callback() + self.assertRaises(Exception, b.Callback) + cb(1) + self.wait() + self.assertEqual(1, val[0]) + + def testEmptyBarrier(self): + val = [False] + def _Callback(result): + self.assertEqual(result, None) + val[0] = True + self.stop() + + with util.MonoBarrier(_Callback): + pass + self.wait() + self.assertTrue(val[0]) + + def testCallbackPositionalArguments(self): + val = [0] + def _Callback(arg1, arg2): + self.stop() + self.assertEqual(arg1, 'arg1') + self.assertEqual(arg2, 'arg2') + val[0] = 1 + + def _Exception(type_, instance_, traceback): + self.stop() + self.assertTrue(type_ is TypeError) + val[0] = 2 + + with util.MonoBarrier(_Callback) as b: + b.Callback()('arg1', 'arg2') + self.wait() + self.assertEqual(1, val[0]) + + with util.Barrier(_Callback, on_exception=_Exception) as b1: + with util.MonoBarrier(_Callback) as b2: + b2.Callback()(b1.Callback()) + self.wait() + self.assertEqual(2, val[0]) + + +class ResultsBarrierTestCase(testing.BaseTestCase): + """Tests for Results barrier.""" + def testResultsBarrier(self): + val = [False] + def _Callback(exp_results, results): + self.stop() + self.assertEqual(results, exp_results) + val[0] = True + + with util.ArrayBarrier(partial(_Callback, [1, 2, 3])) as b: + b.Callback()(1) + b.Callback()(2) + b.Callback()(3) + self.wait() + self.assertTrue(val[0]) + + def testEmptyBarrier(self): + val = [False] + def _Callback(exp_results, results): + self.stop() + self.assertEqual(results, exp_results) + val[0] = True + + with util.ArrayBarrier(partial(_Callback, [])): + pass + self.wait() + self.assertTrue(val[0]) + + def testCompact(self): + val = [False] + def _Callback(exp_results, results): + self.stop() + self.assertEqual(exp_results, results) + val[0] = True + + with util.ArrayBarrier(partial(_Callback, [2]), compact=True) as b: + b.Callback()(None) + b.Callback()(2) + b.Callback()(None) + self.wait() + self.assertTrue(val[0]) + + +class ArrayBarrierTestCase(testing.BaseTestCase): + """Tests for ArrayBarrier barrier type.""" + def testArrayBarrier(self): + val = [False] + def _Callback(exp_results, results): + self.stop() + self.assertEqual(exp_results, results) + val[0] = True + + with util.ArrayBarrier(partial(_Callback, ['cb1', 'cb2', 'cb3', 'cb4'])) as b: + b.Callback()('cb1') + b.Callback()('cb2') + b.Callback()('cb3') + b.Callback()('cb4') + self.wait() + self.assertTrue(val[0]) + + +class DictBarrierTestCase(testing.BaseTestCase): + """Tests for DictBarrier type.""" + def testDictBarrier(self): + val = [False] + def _Callback(exp_results, results): + self.stop() + self.assertEqual(exp_results, results) + val[0] = True + + with util.DictBarrier(partial(_Callback, {'key1': 1, 'key2': 2, 'key3': 3})) as b: + b.Callback('key1')(1) + b.Callback('key2')(2) + b.Callback('key3')(3) + self.wait() + self.assertTrue(val[0]) + + +class ExceptionBarrierTestCase(testing.BaseTestCase): + """Tests for ExceptionBarrier type.""" + def testImmediateException(self): + """Test exception raised before barrier context is exited.""" + def _OnException(type, value, tb): + self.stop() + + with util.ExceptionBarrier(_OnException): + raise Exception('an error') + self.wait() + + def testDelayedException(self): + """Test exception raised after initial barrier context has exited.""" + def _OnException(type, value, tb): + self.stop() + + def _RaiseException(): + raise Exception('an error') + + with util.ExceptionBarrier(_OnException): + self.io_loop.add_callback(_RaiseException) + self.wait() + + def testCallback(self): + """ERROR: Try to use Callback() method on barrier.""" + def _OnException(type, value, tb): + self.assertEqual(type, AssertionError) + self.stop() + + with util.ExceptionBarrier(_OnException) as b: + b.Callback() + self.wait() + + def testMultipleExceptions(self): + """ERROR: Raise multiple exceptions within scope of exception barrier.""" + def _OnException(type, value, tb): + self.stop() + + def _RaiseException(): + raise Exception('an error') + + with util.ExceptionBarrier(_OnException) as b: + self.io_loop.add_callback(_RaiseException) + self.io_loop.add_callback(_RaiseException) + self.wait() + + +class NestedBarrierTestCase(testing.BaseTestCase): + def testUnhandledExeption(self): + """Verify that without an exception handler, a thrown exception + in a barrier propagates. + """ + success = [False] + + def _Op(cb): + raise ZeroDivisionError('exception') + + def _OnSuccess(): + success[0] = True + + def _RunBarrier(): + with util.Barrier(_OnSuccess) as b: + _Op(b.Callback()) + + self.assertRaises(ZeroDivisionError, _RunBarrier) + self.assertTrue(not success[0]) + + def testHandledException(self): + """Verify that if an exception handler is specified, a thrown + exception doesn't propagate. + """ + exception = [False] + success = [False] + + def _OnException(type, value, traceback): + exception[0] = True + self.io_loop.add_callback(self.stop) + + def _OnSuccess(): + success[0] = True + + def _Op(cb): + raise Exception('exception') + + with util.Barrier(_OnSuccess, on_exception=_OnException) as b: + _Op(b.Callback()) + + self.wait() + self.assertTrue(exception[0]) + self.assertTrue(not success[0]) + + def testNestedBarriers(self): + """Verify that a handled exception in a nested barrier doesn't prevent + outer barrier from completing. + """ + exceptions = [False, False] + level1_reached = [False] + + def _Level2Exception(type, value, traceback): + exceptions[1] = True + + def _Level2(cb): + raise Exception('exception in level 2') + + def _Level1Exception(type, value, traceback): + exceptions[0] = True + + def _OnLevel1(): + self.io_loop.add_callback(self.stop) + level1_reached[0] = True + + def _Level1(cb): + with util.Barrier(None, on_exception=_Level2Exception) as b: + _Level2(b.Callback()) + _OnLevel1() + + with util.Barrier(_OnLevel1, on_exception=_Level1Exception) as b: + _Level1(b.Callback()) + self.wait() + self.assertTrue(not exceptions[0]) + self.assertTrue(exceptions[1]) + self.assertTrue(level1_reached[0]) + + +class ParseHostPortTestCase(unittest.TestCase): + def setUp(self): + pass + def tearDown(self): + pass + def testSimple(self): + self.assertEquals(util.ParseHostPort("host:80"), ("host", 80)) + def testSimple2(self): + self.assertEquals(util.ParseHostPort("host.example.com:80"), ("host.example.com", 80)) + def testIP(self): + self.assertEquals(util.ParseHostPort("127.0.0.1:80"), ("127.0.0.1", 80)) + def testEmpty(self): + self.assertRaises(TypeError, util.ParseHostPort, "") + def testHostOnly(self): + self.assertRaises(TypeError, util.ParseHostPort, "host") + def testPortOnly(self): + self.assertRaises(TypeError, util.ParseHostPort, ":1") + def testThreeValues(self): + self.assertRaises(TypeError, util.ParseHostPort, "host:1:2") + def testNoColon(self): + self.assertRaises(TypeError, util.ParseHostPort, "host;1") + def testNonIntegerPort(self): + self.assertRaises(TypeError, util.ParseHostPort, "host:port") + def testOutOfRangePort(self): + self.assertRaises(TypeError, util.ParseHostPort, "host:65536") + def testLongPort(self): + self.assertRaises(TypeError, util.ParseHostPort, "host:1000000000000") + + +class VarLengthEncodeDecodeTestCase(unittest.TestCase): + def testEncode(self): + self._VerifyEncodeDecode(1, '\x01') + self._VerifyEncodeDecode(2, '\x02') + self._VerifyEncodeDecode(127, '\x7f') + self._VerifyEncodeDecode(128, '\x80\x01') + self._VerifyEncodeDecode(255, '\xff\x01') + self._VerifyEncodeDecode(0xffff, '\xff\xff\x03') + self._VerifyEncodeDecode(0xffffffff, '\xff\xff\xff\xff\x0f') + self._VerifyEncodeDecode(0xffffffffffffffff, '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01') + + def testConcatEncodeDecode(self): + numbers = [0xfff112, 0x12, 0x0, 0xffffffffff] + raw_bytes = '' + for n in numbers: + raw_bytes += util.EncodeVarLengthNumber(n) + for n in numbers: + val, length = util.DecodeVarLengthNumber(raw_bytes) + self.assertEqual(val, n) + raw_bytes = raw_bytes[length:] + + def testInvalidDecode(self): + self.assertRaises(TypeError, util.DecodeVarLengthNumber, '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff') + + def _VerifyEncodeDecode(self, number, string): + self.assertEqual(util.EncodeVarLengthNumber(number), string) + self.assertEqual(util.DecodeVarLengthNumber(string), (number, len(string))) + + +class DecayingStatTestCase(unittest.TestCase): + def testDecay(self): + now = 0.0 + stat = util.DecayingStat(half_life=1.0, now=now) + stat.Add(1.0, now) + self.assertAlmostEquals(stat.Get(now), 1.0) + stat.Add(1.0, now) + self.assertAlmostEquals(stat.Get(now), 2.0) + now = 1.0 + self.assertAlmostEquals(stat.Get(now), 1.0) + + +class LRUCacheTestCase(unittest.TestCase): + def testExpiration(self): + cache = util.LRUCache(4) + + # Populate the cache + self.assertEqual(cache.Get(1, lambda: 1), 1) + self.assertEqual(cache.Get(2, lambda: 2), 2) + self.assertEqual(cache.Get(3, lambda: 3), 3) + self.assertEqual(cache.Get(4, lambda: 4), 4) + + # Access 2 and 1 to move them to the top (and see that they are not yet evicted, so the + # factory function is ignored) + self.assertEqual(cache.Get(2, lambda: None), 2) + self.assertEqual(cache.Get(1, lambda: None), 1) + + # Add a fifth object and see #3 get evicted: + self.assertEqual(cache.Get(5, lambda: 5), 5) + self.assertEqual(cache.Get(3, lambda: None), None) + + +class ThrottleRateTestCase(unittest.TestCase): + def testThrottle(self): + util._TEST_TIME = time.time() + + # Null and empty cases. + self.assertEqual(util.ThrottleRate(None, 1, 1), ({'count': 1, 'start_time': util._TEST_TIME}, False)) + self.assertEqual(util.ThrottleRate({}, 1, 1), ({'count': 1, 'start_time': util._TEST_TIME}, False)) + + # Increment existing. + self.assertEqual(util.ThrottleRate({'count': 1, 'start_time': util._TEST_TIME}, 2, 1), + ({'count': 2, 'start_time': util._TEST_TIME}, False)) + + # Reset existing. + self.assertEqual(util.ThrottleRate({'count': 10, 'start_time': util._TEST_TIME - 1}, 1, 1), + ({'count': 1, 'start_time': util._TEST_TIME}, False)) + + # Exceed. + self.assertEqual(util.ThrottleRate({}, 0, 1), ({'count': 0, 'start_time': util._TEST_TIME}, True)) + self.assertEqual(util.ThrottleRate({'count': 1, 'start_time': util._TEST_TIME}, 1, 1), + ({'count': 1, 'start_time': util._TEST_TIME}, True)) diff --git a/backend/base/testing.py b/backend/base/testing.py new file mode 100644 index 0000000..f6ad9a1 --- /dev/null +++ b/backend/base/testing.py @@ -0,0 +1,265 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Base testing utilities. + +Make sure to decorate all asynchronous test methods with @async_test. + + @async_test: use this decorator for all asynchronous tests + @async_test_timeout(timeout=<>): use this decorator for all asynchronous tests to set timeout +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + + +import functools +import logging +import re +import sys +import threading +import time +import unittest + + +from cStringIO import StringIO +from functools import partial +from tornado.concurrent import Future +from tornado import testing, ioloop, httpclient + + +def async_test(method): + """Decorator for tests running in test cases derived from + tornado.testing.AsyncTestCase and tornado.testing.AsyncHTTPTestCase. + + On error, calls self.stop(). + + NOTE: Tests are responsible for calling self.stop() to signal to + tornado async test framework that test is complete. + """ + @functools.wraps(method) + def Wrapper(self): + method(self) + self.wait() + + return Wrapper + +# Tell nose that this method named _test isn't a test. +async_test.__test__ = False + +class TestThread(threading.Thread): + """Thread class which runs the specified test method. On + completion, invokes the completion method. + """ + def __init__(self, test_method, on_completion): + super(TestThread, self).__init__() + self._test_method = test_method + self._on_completion = on_completion + self._exc_info = None + + def run(self): + try: + self._test_method() + except: + self._exc_info = sys.exc_info() + finally: + self._on_completion() + + def MaybeRaise(self): + if self._exc_info is not None: + type, value, tb = self._exc_info + raise type, value, tb + + +def thread_test(method): + """Decorator for tests which need to be run synchronously. Runs + the test in a separate thread. + """ + @functools.wraps(method) + def Wrapper(self): + thread = TestThread(partial(method, self), + partial(self.io_loop.add_callback, self.stop)) + thread.start() + self.wait() + thread.MaybeRaise() + + return Wrapper + +thread_test.__test__ = False + +def async_test_timeout(timeout=5): + def _async_test(method): + def _wrapper(self, *args, **kwargs): + method(self, *args, **kwargs) + self.wait(timeout=timeout) + + return functools.wraps(method)(_wrapper) + return _async_test + + +class BaseTestCase(testing.AsyncTestCase): + """Base TestCase class for simple asynchronous tests. + This class can be used as a mix-in in conjunction with tornado's AsyncHTTPTestCase. + """ + def wait(self, condition=None, timeout=5): + """It is convenient to repeatedly use the "wait" method in order to + create synchronous tests. If the async call raises an exception, then + the wait method will re-raise that exception, which is desirable. + However, when this happens, Tornado "remembers" the exception, and + will re-throw it *every* time that wait is called from then on. This + override patches this behavior by clearing the private __failure + field in Tornado's testing class so that subsequent waits will not + fail. + """ + try: + return super(BaseTestCase, self).wait(condition, timeout) + finally: + self._AsyncTestCase__failure = None + + def _RunAsync(self, func, *args, **kwargs): + """Runs an async function which takes a callback argument. Waits for + the function to complete and returns any result. + """ + func(callback=self.stop, *args, **kwargs) + return self.wait() + + +class LogMatchTestCase(testing.LogTrapTestCase): + """Mix in the methods of this class in order to intercept and possibly + test the content of logs that are produced by the test case. This + class turns on all logging levels and adds a convenient assert method + that can be used to test the output of the logging for desired patterns. + """ + def run(self, result=None): + """Override the run method in order to set the root logger to the + NOTSET logging level, so that no logging done by the test case will + be suppressed. Restore the original logging level once the test + case has been run. + """ + logger = logging.getLogger() + current_level = logger.level + try: + logger.setLevel('NOTSET') + super(LogMatchTestCase, self).run(result) + finally: + logger.setLevel(current_level) + + def assertLogMatches(self, expected_regexp, msg=None): + """Fail the test unless the intercepted log matches the regular + expression. + """ + format = '%s: %%r was not found in log' % (msg or 'Regexp didn\'t match') + self._AssertLogMatches(expected_regexp, False, format) + + def assertNotLogMatches(self, expected_regexp, msg=None): + """Fail the test if the intercepted log *does* match the regular + expression. + """ + format = '%s: %%r was found in log' % (msg or 'Regexp matches') + self._AssertLogMatches(expected_regexp, True, format) + + def _AssertLogMatches(self, expected_regexp, invert, format): + """Assert if the intercepted log matches the regular expression, + or if "invert" is True and no match is found. + """ + if isinstance(expected_regexp, basestring): + expected_regexp = re.compile(expected_regexp, re.MULTILINE | re.DOTALL) + + handler = logging.getLogger().handlers[0] + if not hasattr(handler, 'stream'): + # Nose's test runner installs a log handler that this test isn't compatible with. + # TODO(ben): figure out how to make this work. + raise unittest.SkipTest() + log_text = handler.stream.getvalue() + matches = expected_regexp.search(log_text) is not None + if matches == invert: + raise self.failureException(format % expected_regexp.pattern) + +class TimingTextTestResult(unittest.TextTestResult): + def startTest(self, test): + self.__start_time = time.time() + super(TimingTextTestResult, self).startTest(test) + + def addSuccess(self, test): + if self.showAll: + delta = time.time() - self.__start_time + self.stream.write('(%d ms) ' % int(delta * 1000)) + super(TimingTextTestResult, self).addSuccess(test) + +class TimingTextTestRunner(unittest.TextTestRunner): + """Wraps the standard unittest runner to print additional information. + + In verbose mode, each test prints how long it took to run. Also + prints the class and function name instead of the docstring by default. + """ + def __init__(self, *args, **kwargs): + kwargs.setdefault('resultclass', TimingTextTestResult) + kwargs.setdefault('descriptions', False) + super(TimingTextTestRunner, self).__init__(*args, **kwargs) + +# not a subclass of AsyncHTTPClient to avoid __new__ magic +class MockAsyncHTTPClient(object): + """Mock HTTP client for tests. + + Has the same interface as `tornado.httpclient.AsyncHTTPClient`, + but returns a pre-configured response to any request. + To use, create a MockAsyncHTTPClient, call `map` at least once + to map urls to responses, then use `fetch` to make the requests. + + While it is recommended that any code using an AsyncHTTPClient + allow it to be passed in as an argument, we have a lot of code + that relies on AsyncHTTPClient's magic pseudo-singleton behavior + and "constructs" a new client each time. For this, we support + `mock.patch`: + + with mock.patch('tornado.httpclient.AsyncHTTPClient', MockAsyncHTTPClient() as mock_client: + mock_client.map(url, response) + call_other_functions() + + Note that this is a non-standard use of patch (we're replacing a class + with an instance); this is somewhat hacky but the simplest way of + dealing with the instantiation magic in AsyncHTTPClient. + """ + def __init__(self, io_loop=None): + self.io_loop = io_loop or ioloop.IOLoop.current() + self.url_map = [] + + def map(self, regex, response): + """Maps a url regex to a response. + + Any request whose url matches the given regex will get the corresponding + response (if multiple regexes match, the most recently mapped one wins). + The response may be a string (used as the body), a + `tornado.httpclient.HTTPResponse` object, or a function that takes a + request and returns one of the preceding types. + """ + self.url_map.insert(0, (re.compile(regex), response)) + + def fetch(self, request, callback=None, **kwargs): + """Implementation of AsyncHTTPClient.fetch""" + if not isinstance(request, httpclient.HTTPRequest): + request = httpclient.HTTPRequest(url=request, **kwargs) + for regex, response in self.url_map: + if regex.match(request.url): + if callable(response): + response = response(request) + if isinstance(response, basestring): + response = httpclient.HTTPResponse(request, 200, + buffer=StringIO(response)) + assert isinstance(response, httpclient.HTTPResponse) + if callback is not None: + self.io_loop.add_callback(functools.partial(callback, response)) + return None + else: + future = Future() + future.set_result(response) + return future + raise ValueError("got request for unmapped url: %s" % request.url) + + def __call__(self, io_loop=None): + """Hacky support for mock.patch. + + We patch the AsyncHTTPClient class and replace it with this instance, + so "calling" the instance should act like instantiating the class. + """ + if io_loop is not None: + assert io_loop is self.io_loop + return self diff --git a/backend/base/thrift_transport.py b/backend/base/thrift_transport.py new file mode 100644 index 0000000..ebc691d --- /dev/null +++ b/backend/base/thrift_transport.py @@ -0,0 +1,206 @@ +"""Non-blocking thrift (http://thrift.apache.com) transport. + +This is an implementation of thrift's TTransportBase interface for use +with a Tornado web server. It uses an ioloop.IOLoop object for +processing async I/O, and greenlets for coroutine functionality used +to seamlessly jump the thread of execution from the blocking, +synchronous thrift call stack back out to the normal web application. +Callbacks registered with the IOLoop are invoked on thrift socket +activity, causing the paused greenlet to resume and continue +processing. When complete, the original greenlet will unwind its call +stack and return execution to the point where the original thrift call +was made. + +This transport (indeed, any thrift transport) is not thread-safe. + +Based on code in thrift/transport/TSocket.py and tornado/simple_httpclient.py +and on the explanation of marrying Tornado to Boto by Josh Haas: + +http://blog.joshhaas.com/2011/06/marrying-boto-to-tornado-greenlets-bring-them-together/ + + TTornadoTransport: asynchronous thrift client transport using IOLoop +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + + +import functools +import greenlet +import logging +import socket +import time + +from thrift.transport.TTransport import TTransportBase, TTransportException + +from tornado import ioloop +from tornado.iostream import IOStream + + +def _wrap_transport(method): + """Decorator to consistently check the underlying stream, setup the + on-close callback, and create & remove a timeout expiration, if + applicable. + """ + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + self._check_stream() + self._stream.set_close_callback(functools.partial( + self._on_close, gr=greenlet.getcurrent())) + self._start_time = time.time() + timeout = self._set_timeout() + try: + return method(self, *args, **kwargs) + except TTransportException: + self.close() + raise + finally: + self._clear_timeout(timeout) + if self._stream: + self._stream.set_close_callback(functools.partial( + self._on_close, gr=None)) + return wrapper + + +class TTornadoTransport(TTransportBase): + """A non-blocking Thrift client. + + Example usage:: + + import greenlet + from tornado import ioloop + from thrift.transport import TTransport + from thrift.protocol import TBinaryProtocol + + from viewfinder.backend.thrift import TTornadoTransport + + transport = TTransport.TFramedTransport(TTornadoTransport('localhost', 9090)) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + client = Service.Client(protocol) + ioloop.IOLoop.instance().start() + + Then, from within an asynchronous tornado request handler: + + class MyApp(tornado.web.RequestHandler): + @tornado.web.asynchronous + def post(self): + def business_logic(): + ...any thrift calls... + self.write(...stuff that gets returned to client...) + self.finish() #end the asynchronous request + gr = greenlet.greenlet(business_logic) + gr.switch() + """ + + def __init__(self, host='localhost', port=9090): + """Initialize a TTornadoTransport with a Tornado IOStream. + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + """ + self.host = host + self.port = port + self._stream = None + self._io_loop = ioloop.IOLoop.current() + self._timeout_secs = None + + def set_timeout(self, timeout_secs): + """Sets a timeout for use with open/read/write operations.""" + self._timeout_secs = timeout_secs + + def isOpen(self): + return self._stream is not None + + def open(self): + """Creates a connection to host:port and spins up a tornado + IOStream object to write requests and read responses from the + thrift server. After making the asynchronous connect call to + _stream, the current greenlet yields control back to the parent + greenlet (presumably the "master" greenlet). + """ + assert greenlet.getcurrent().parent is not None + # TODO(spencer): allow ipv6? (af = socket.AF_UNSPEC) + addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_INET, + socket.SOCK_STREAM, 0, 0) + af, socktype, proto, canonname, sockaddr = addrinfo[0] + self._stream = IOStream(socket.socket(af, socktype, proto), + io_loop=self._io_loop) + self._open_internal(sockaddr) + + def close(self): + if self._stream: + self._stream.set_close_callback(None) + self._stream.close() + self._stream = None + + @_wrap_transport + def read(self, sz): + logging.debug("reading %d bytes from %s:%d" % (sz, self.host, self.port)) + cur_gr = greenlet.getcurrent() + def _on_read(buf): + if self._stream: + cur_gr.switch(buf) + self._stream.read_bytes(sz, _on_read) + buf = cur_gr.parent.switch() + if len(buf) == 0: + raise TTransportException(type=TTransportException.END_OF_FILE, + message='TTornadoTransport read 0 bytes') + logging.debug("read %d bytes in %.2fms" % + (len(buf), (time.time() - self._start_time) * 1000)) + return buf + + @_wrap_transport + def write(self, buf): + logging.debug("writing %d bytes to %s:%d" % (len(buf), self.host, self.port)) + cur_gr = greenlet.getcurrent() + def _on_write(): + if self._stream: + cur_gr.switch() + self._stream.write(buf, _on_write) + cur_gr.parent.switch() + logging.debug("wrote %d bytes in %.2fms" % + (len(buf), (time.time() - self._start_time) * 1000)) + + @_wrap_transport + def flush(self): + pass + + @_wrap_transport + def _open_internal(self, sockaddr): + logging.debug("opening connection to %s:%d" % (self.host, self.port)) + cur_gr = greenlet.getcurrent() + def _on_connect(): + if self._stream: + cur_gr.switch() + self._stream.connect(sockaddr, _on_connect) + cur_gr.parent.switch() + logging.info("opened connection to %s:%d" % (self.host, self.port)) + + def _check_stream(self): + if not self._stream: + raise TTransportException( + type=TTransportException.NOT_OPEN, message='transport not open') + + def _set_timeout(self): + if self._timeout_secs: + return self._io_loop.add_timeout( + time.time() + self._timeout_secs, functools.partial( + self._on_timeout, gr=greenlet.getcurrent())) + return None + + def _clear_timeout(self, timeout): + if timeout: + self._io_loop.remove_timeout(timeout) + + def _on_timeout(self, gr): + gr.throw(TTransportException( + type=TTransportException.TIMED_OUT, + message="connection timed out to %s:%d" % (self.host, self.port))) + + def _on_close(self, gr): + self._stream = None + message = "connection to %s:%d closed" % (self.host, self.port) + if gr: + gr.throw(TTransportException( + type=TTransportException.NOT_OPEN, message=message)) + else: + logging.error(message) diff --git a/backend/base/util.py b/backend/base/util.py new file mode 100644 index 0000000..3c9f741 --- /dev/null +++ b/backend/base/util.py @@ -0,0 +1,681 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Utility methods. + + Barrier: synchronization barrier with no results + MonoBarrier: synchronization barrier that returns a single result + ArrayBarrier: synchronization barrier with ordered results + DictBarrier: synchronization barrier with dictionary of results + ParseJSONResponse(): parses a JSON response body into python data structures + ParseHostPort(): parses host:port string and returns tuple +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import base64 +import calendar +import collections +import hashlib +import json +import logging +import math +import os +import pwd +import random +import re +import struct +import sys +import time + +from datetime import date, datetime, timedelta +from functools import partial +from tornado import gen, ioloop, stack_context +from viewfinder.backend.base import base64hex + + +def NoCallback(*args, **kwargs): + """Used where the conclusion of an asynchronous operation can be + ignored. + """ + pass + + +def LogExceptionCallback(type, value, tb): + """Logs and then ignores the exception.""" + logging.error('unexpected error', exc_info=(type, value, tb)) + + +class _BarrierContext(object): + """A context manager which returns a synchronization barrier for + managing any number of asynchronous operations started within its + context. A final callback is specified on construction to be invoked + after all callbacks allocated during the __enter__ phase of the + barrier context have been invoked. + + If an exception is thrown at any time during the execution of the + barrier (including in its entire hierarchical sub tree of + asynchronous operations), the barrier is aborted and will never + finish. Its current batch of results and any incoming results are + discarded. The barrier completion callback will never be run. Only + the first exception is acted upon; all others thrown in subsequent + processing of extant async execution pathways are logged tersely. + + If 'on_exception' is specified, it is invoked to handle intentional + cleanup and/or to continue processing along an alternate (error + recovery) execution pathway. The stack context that existed when the + barrier was created is restored in order to run 'on_exception'. If + 'on_exception' would like to propagate the exception, the exception + should be re-raised. The exception handler can optionally return a + boolean value to indicate whether or not the underlying exception + should be logged. + """ + def __init__(self, callback, barrier_type, on_exception=None): + callback = stack_context.wrap(callback) + on_exception = stack_context.wrap(on_exception) + # Parent frame is derived class __init__, so get grandparent frame. + frame = sys._getframe().f_back.f_back + self._barrier = _Barrier(callback, on_exception, barrier_type, frame) + self._stack_context = stack_context.ExceptionStackContext(self._barrier.ReportException) + + def __enter__(self): + self._stack_context.__enter__() + return self._barrier + + def __exit__(self, type, value, traceback): + if (type, value, traceback) == (None, None, None): + self._barrier.Start() + return self._stack_context.__exit__(type, value, traceback) + + +class Barrier(_BarrierContext): + """Barrier that discards all return values and invokes final + callback without arguments. + """ + def __init__(self, callback, on_exception=None): + super(Barrier, self).__init__(callback, _Barrier.BARRIER, + on_exception=on_exception) + + +class MonoBarrier(_BarrierContext): + """Barrier that expects and returns a single value from a single + constituent op, or None, if no ops are run. + """ + def __init__(self, callback, on_exception=None): + super(MonoBarrier, self).__init__(callback, _Barrier.MONO_BARRIER, + on_exception=on_exception) + + +class ArrayBarrier(_BarrierContext): + """Barrier that returns results from asynchronous ops as an + ordered list. + """ + def __init__(self, callback, compact=False, on_exception=None): + super(ArrayBarrier, self).__init__( + callback, _Barrier.COMPACT_ARRAY_BARRIER if compact else _Barrier.ARRAY_BARRIER, + on_exception=on_exception) + + +class DictBarrier(_BarrierContext): + """Barrier that returns results from asynchronous ops as a dict. + """ + def __init__(self, callback, on_exception=None): + super(DictBarrier, self).__init__(callback, _Barrier.DICT_BARRIER, + on_exception=on_exception) + + +class ExceptionBarrier(_BarrierContext): + """Barrier that handles exceptions by routing the first exception to + the 'on_exception' handler and logging any subsequent exceptions. + """ + def __init__(self, on_exception): + super(ExceptionBarrier, self).__init__(NoCallback, _Barrier.EXC_BARRIER, + on_exception=on_exception) + + +class _Barrier(object): + """Provides a synchronization barrier for invoking a final callback + when the barrier has been invoked the specified number of times. + Each invocation of the barrier expects a result object which is + appended to a final list. The final list is passed to the barrier + callback. + """ + BARRIER = 0 + MONO_BARRIER = 1 + ARRAY_BARRIER = 2 + COMPACT_ARRAY_BARRIER = 3 + DICT_BARRIER = 4 + EXC_BARRIER = 5 + + _types = { + BARRIER: 'barrier', + MONO_BARRIER: 'mono barrier', + ARRAY_BARRIER: 'array barrier', + COMPACT_ARRAY_BARRIER: 'compact array barrier', + DICT_BARRIER: 'dict barrier', + EXC_BARRIER: 'exception barrier', + } + + _INITIALIZING = 0 + """Callbacks are being accumulated in initial "with" statement.""" + + _STARTED = 1 + """Initial "with" statement is complete; now waiting for async callbacks.""" + + _COMPLETED = 2 + """All async callbacks have been invoked successfully.""" + + _FAULTED = 3 + """An exception occurred during initialization or async execution.""" + + def __init__(self, callback, on_exception, barrier_type, frame): + self._callback = callback + self._on_exception = on_exception + self._type = barrier_type + self._filename = os.path.basename(frame.f_code.co_filename) + self._lineno = frame.f_lineno + self._n = 0 + self._cur = 0 + self._state = _Barrier._INITIALIZING + if self._type == _Barrier.BARRIER: + self._results = None + elif self._type == _Barrier.MONO_BARRIER: + self._results = None + elif self._type in (_Barrier.ARRAY_BARRIER, _Barrier.COMPACT_ARRAY_BARRIER): + self._results = [] + elif self._type == _Barrier.DICT_BARRIER: + self._results = {} + else: + assert self._type == _Barrier.EXC_BARRIER + self._results = None + logging.debug('constructed %s', _Barrier._types[barrier_type]) + + def Callback(self, key=None): + """Returns a callback upon which the barrier will be gated. For + completion, every callback returned via invocations of this method + must be invoked. If 'key' is not None, the result returned with + the callback will be a dict. Otherwise, results (if any) will be + returned as an ordered list. + """ + assert self._type != _Barrier.EXC_BARRIER, \ + 'exception barriers do not have results' + + if key is not None: + assert self._type == _Barrier.DICT_BARRIER, \ + 'this barrier is not configured as a dictionary of results' + else: + if self._type == _Barrier.ARRAY_BARRIER: + self._results.append(None) + key = self._cur + + if self._type == _Barrier.MONO_BARRIER: + assert self._cur == 0, 'mono-barrier cannot return multiple results' + + self._cur += 1 + self._n += 1 + return partial(self._Invoke, key) + + def Start(self): + """Invoked when all constituent async ops which this barrier is + gated on have been launched. This is called from the + BarrierContext's __exit__ method. + """ + logging.debug('starting %s with %d async execution pathways...', + _Barrier._types[self._type], self._cur) + assert self._state == _Barrier._INITIALIZING, 'barrier was already started' + self._state = _Barrier._STARTED + self._MaybeReturn() + + def ReportException(self, type, value, tb): + """Called when an exception occurs during initialization or during + async op execution. Discards all results, transitions the barrier + to the FAULTED state, and invokes the '_on_exception' callback. + Returns True if the exception should not be propagated further. + """ + if self._state == _Barrier._INITIALIZING or self._state == _Barrier._STARTED: + self._state = _Barrier._FAULTED + self._results = None + self._callback = None + + if self._on_exception is not None: + ioloop.IOLoop.current().add_callback(self._on_exception, type, value, tb) + return True + else: + logging.info('exception in barrier (%s) with no exception handler: %s; propagating...' % + (self._FormatBarrierLocation(), FormatLogArgument(value))) + return False + elif self._state == _Barrier._COMPLETED: + logging.error('exception in barrier (%s) that is already completed; propagating...' % + self._FormatBarrierLocation(), exc_info=(type, value, tb)) + return False + + assert self._state == _Barrier._FAULTED, self._state + logging.info('more than one exception in barrier (%s): %s; ignoring...' % + (self._FormatBarrierLocation(), FormatLogArgument(value))) + return True + + def _FormatBarrierLocation(self): + """Return a human-readable format of the location of the barrier in + the source code. + """ + return '%s:%d' % (self._filename, self._lineno) + + def _Invoke(self, *args): + """Result callback for constituent asynchronous operations which + the barrier is gated on. The results are aggregated in self._result + based on '*args'. + """ + assert len(args) >= 1, args + key = args[0] + val = None + if len(args) > 1: + val = args[1] if len(args) == 2 else args[1:] + + if self._state == _Barrier._FAULTED: + logging.info('discarding result %r intended for faulted barrier (%d): %r' % + (key, self._cur, FormatLogArgument(val))) + return + + assert self._state != _Barrier._COMPLETED, 'Why still getting results after completion?' + + if self._type == _Barrier.COMPACT_ARRAY_BARRIER: + if val is not None: + self._results.append(val) + else: + logging.debug('discarding empty result') + elif self._type == _Barrier.MONO_BARRIER: + assert self._results is None, self._results + self._results = val + elif self._type != _Barrier.BARRIER: + self._results[key] = val + self._n -= 1 + assert self._n >= 0, 'barrier invoked more than %d times Callback() was invoked' % self._cur + self._MaybeReturn() + + def _MaybeReturn(self): + """Schedules the barrier callback if the barrier has been started + and all results have been received. If all results were empty, + schedules the barrier callback with no arguments. If the barrier is + a mono barrier, and the results are a list, schedules the callback with + the list expanded into positional arguments. The callback is scheduled, + rather than invoked directly, to ensure that any contexts established + inside the scope of the barrier are properly unrolled before the + callback is invoked. + """ + if self._n == 0 and self._state == _Barrier._STARTED and self._type != _Barrier.EXC_BARRIER: + self._state = _Barrier._COMPLETED + + callback = self._callback + self._callback = None + + results = self._results + self._results = None + + if self._type == _Barrier.BARRIER: + logging.debug('%s finished', _Barrier._types[self._type]) + if callback is not None: + ioloop.IOLoop.current().add_callback(callback) + elif self._type == _Barrier.MONO_BARRIER and type(results) == tuple: + logging.debug('%s finished with tuple result: %r', _Barrier._types[self._type], results) + if callback is not None: + ioloop.IOLoop.current().add_callback(callback, *results) + else: + logging.debug('%s finished with result: %r', _Barrier._types[self._type], results) + if callback is not None: + ioloop.IOLoop.current().add_callback(callback, results) + + +_TEST_TIME = None +"""If not None, _TEST_TIME is used in various places in the server where +time.time() is normally called. This value is overridden by tests that +require determinism. +""" + + +def GetCurrentTimestamp(): + """If _TEST_TIME has been set, returns it. Otherwise, returns the + current timestamp. Doing this enables tests to be more deterministic. + """ + return _TEST_TIME if _TEST_TIME is not None else time.time() + + +def CreateSortKeyPrefix(timestamp, randomness=True, reverse=False): + """Returns a sort key which will sort by 'timestamp'. If + 'randomness' is True, 16 bits of randomness (which would otherwise + be lost to b64-encoding padding) are added into the free bits. These + are meant to minimize the chance of collision when the sort key + prefix is meant to provide uniqueness but many keys may be created + in the same second. If 'reverse' is True, the timestamp is reversed + by subtracting from 2^32. The result is base64hex-encoded. + """ + assert timestamp < 1L << 32, timestamp + if reverse: + timestamp = (1L << 32) - int(timestamp) - 1 + if randomness: + random_bits = random.getrandbits(16) & 0xffff + else: + random_bits = 0 + return base64hex.B64HexEncode(struct.pack( + '>IH', int(timestamp), random_bits)) + + +def UnpackSortKeyPrefix(prefix): + """Returns the timestamp in the provided sort key prefix. The + timestamp may be reversed, if reverse=True was specified when + the sort key prefix was created. + """ + timestamp, random_bits = struct.unpack('>IH', base64hex.B64HexDecode(prefix)) + return timestamp + + +def ParseHostPort(address): + """Parses the provided address string as host:port and + returns a tuple of (str host, int port). + """ + host_port_re = re.match(r"([a-zA-Z0-9-\.]+):([0-9]{1,5})$", address) + if not host_port_re: + raise TypeError("bad host:port: %s" % address) + host = host_port_re.group(1) + port = int(host_port_re.group(2)) + if port >= 65536: + raise TypeError("invalid port: %d" % port) + return (host, port) + + +def EncodeVarLengthNumber(value): + """Uses a variable length encoding scheme. The first 7 bits of each + byte contain the value and the 8th bit is a continuation bit. Returns + a string of raw bytes. Supply the output to DecodeVarLengthNumber to + decode and retrieve the original number. + """ + byte_str = '' + while value >= 128: + byte_str += struct.pack('>B', (value & 127) | 128) + value >>= 7 + byte_str += struct.pack('>B', value & 127) + return byte_str + + +def DecodeVarLengthNumber(byte_str): + """Interprets a raw byte string as a variable length encoded number + and decodes. Returns a pair consisting of the decoded number and the + number of bytes consumed to decode the number. + """ + value = 0 + num_bytes = 0 + for shift in xrange(0, 64, 7): + byte, = struct.unpack('>B', byte_str[num_bytes:num_bytes + 1]) + num_bytes += 1 + if byte & 128: + value |= ((byte & 127) << shift) + else: + value |= (byte << shift) + return value, num_bytes + raise TypeError('string not decodable as variable length number') + + +class DecayingStat(object): + """Decaying stat class which exponentially decays stat measurements + according to a 'half-life' setting specified to the constructor. + """ + def __init__(self, half_life, now=None): + """The 'half_life' is specified in seconds.""" + self._half_life = half_life + self._last_time = now if now is not None else time.time() + self._value = 0 + + def Add(self, value, now=None, ceiling=None): + now = now if now is not None else time.time() + self._value = self._Decay(self._value, now) + value + if ceiling is not None: + self._value = min(ceiling, self._value) + self._last_time = now + + def Get(self, now=None): + now = now if now is not None else time.time() + return self._Decay(self._value, now) + + def _Decay(self, value, now): + delta_secs = now - self._last_time + if delta_secs == 0.0: + return value + return value * math.exp(-math.log(2.0) * delta_secs / self._half_life) + + +class GenConstant(gen.YieldPoint): + """Yields a constant value. Used with the tornado.gen infrastructure.""" + def __init__(self, constant): + self._constant = constant + + def start(self, runner): + pass + + def is_ready(self): + return True + + def get_result(self): + return self._constant + + +def GenSleep(seconds): + """Wait for a period of time without blocking. Used with the tornado.gen infrastructure.""" + io_loop = ioloop.IOLoop.current() + return gen.Task(io_loop.add_timeout, io_loop.time() + seconds) + + +def FormatLogArgument(s): + """Format "s" in a human-readable way for logging by truncating it + to at most 256 characters. + """ + MAX_LEN = 256 + if isinstance(s, unicode): + s = s.encode('utf-8') + else: + s = str(s) + if len(s) <= MAX_LEN: + return s + return '%s...[%d bytes truncated]' % (s[:MAX_LEN], len(s) - MAX_LEN) + + +def FormatArguments(*args, **kwargs): + """Format function call arguments in a human-readable way for logging.""" + def _FormatArg(arg): + return FormatLogArgument(arg) + + # Truncate arguments. + args = [_FormatArg(arg) for arg in args] + kwargs = {key: _FormatArg(value) for key, value in kwargs.items()} + + return "(args=%s, kwargs=%s)" % (args, kwargs) + + +def FormatFunctionCall(func, *args, **kwargs): + """Format a function and its arguments in a human-readable way for logging.""" + while type(func) is partial: + args = func.args + args + if func.keywords: + kwargs.update(func.keywords) + func = func.func + + return "%s%s" % (func.__name__, FormatArguments(*args, **kwargs)) + + +def TimestampUTCToISO8601(timestamp): + """Return the timestamp (UTC) ISO 8601 format: YYYY-MM-DD.""" + utc_tuple = datetime.utcfromtimestamp(timestamp) + return '%0.4d-%0.2d-%0.2d' % (utc_tuple.year, utc_tuple.month, utc_tuple.day) + + +def NowUTCToISO8601(): + """Return the current date (UTC) in ISO 8601 format: YYYY-MM-DD.""" + return TimestampUTCToISO8601(time.time()) + + +def ISO8601ToUTCTimestamp(day, hour=0, minute=0, second=0): + """Convert the day in ISO 8601 format (YYYY-MM-DD) to timestamp.""" + y, m, d = day.split('-') + return calendar.timegm(datetime(int(y), int(m), int(d), hour, minute, second).timetuple()) + + +def SecondsSince(timestamp): + """Seconds since a given timestamp.""" + return time.time() - timestamp + + +def HoursSince(timestamp): + """Hours since a given timestamp. Floating point.""" + return SecondsSince(timestamp) / 3600. + + +def GetSingleListItem(list, default=None): + """Return the first item in the list, or "default" if the list is None + or empty. Assert that the list contains at most one item. + """ + if list: + assert len(list) == 1, list + return list[0] + return default + + +def Pluralize(count, singular='', plural='s'): + """Return the pluralization suffix for "count" item(s). For example: + 'item' + Pluralize(1) = 'item' + 'item' + Pluralize(2) = 'items' + 'activit' + Pluralize(1, 'y', 'ies') = 'activity' + 'activit' + Pluralize(0, 'y', 'ies') = 'activities' + """ + return singular if count == 1 else plural + + +def ComputeMD5Hex(byte_str): + """Compute MD5 hash of "byte_str" and return it encoded as hex string.""" + hasher = hashlib.md5() + hasher.update(byte_str) + return hasher.hexdigest() + + +def ComputeMD5Base64(byte_str): + """Compute MD5 hash of "byte_str" and return it encoded as a base-64 + string. + """ + hasher = hashlib.md5() + hasher.update(byte_str) + return base64.b64encode(hasher.digest()) + + +def ToCanonicalJSON(dict, indent=False): + """Convert "dict" to a canonical JSON string. Sort keys so that output + ordering is always the same. + """ + return json.dumps(dict, sort_keys=True, indent=indent) + + +def SetIfNotNone(dict, attr_name, value): + """If "value" is not None, then set the specified attribute of "dict" + to its value. + """ + if value is not None: + dict[attr_name] = value + + +def SetIfNotEmpty(dict, attr_name, value): + """If "value" is not empty and non-zero, then set the specified attribute of "dict" to + its value. + """ + if value: + dict[attr_name] = value + + +def ConvertToString(value): + """Converts value, if a number, to a string.""" + if type(value) in (int, long): + return str(value) + elif type(value) == float: + # Need to use repr(), since str() rounds floats. However, can't use repr() for longs, + # since it adds an "L". + return repr(value) + else: + return value + + +def ConvertToNumber(value): + """Converts value, a string, into either an integer or a floating point + decimal number. + """ + try: + # int() automatically promotes to long if necessary + return int(value) + except: + return float(value) + + +class LRUCache(object): + """Simple LRU cache. + + Usage: + value = cache.Get(key, lambda: CreateValue(...)) + + The factory function will be called only if the value does not exist in the cache. + """ + def __init__(self, max_size): + self._max_size = max_size + self._cache = collections.OrderedDict() + + def Get(self, key, factory): + if key in self._cache: + # Delete and re-add the object to move it to the top of the list. + value = self._cache.pop(key) + else: + value = factory() + self._cache[key] = value + while len(self._cache) > self._max_size: + self._cache.popitem(False) + return value + + +def CheckRequirements(filename): + """Parse a pip requirements.txt and raise an exception if any packages + are not installed. + """ + from pip.req import parse_requirements + errors = [] + for req in parse_requirements(filename): + req.check_if_exists() + if not req.satisfied_by: + errors.append(req) + if errors: + raise RuntimeError("Requirements not installed: %s" % [str(e) for e in errors]) + + +def GetLocalUser(): + """Return the local user running the program. + os.getlogin() fails on ubuntu cron jobs, hence the getuid method. + """ + return pwd.getpwuid(os.getuid())[0] or os.getlogin() + + +def ThrottleRate(throttle_dict, max_count, time_period): + """Throttle the rate at which occurrences of some event can occur. The maximum rate is given + as a period of time and the max number of occurrences that can happen within that period. + + The current count and elapsed time is stored in "throttle_dict": + - start_time: time at which the current period started + - count: number of occurrences so far in the current period + + Returns a 2-tuple containing an updated throttle_dict and a boolean indicating whether the + occurrence rate has exceeded the maximum allowed: + (throttle_dict, is_throttled) + """ + now = GetCurrentTimestamp() + if not throttle_dict or now >= throttle_dict['start_time'] + time_period: + throttle_dict = {'start_time': now, + 'count': 0} + else: + throttle_dict = {'start_time': throttle_dict['start_time'], + 'count': throttle_dict['count']} + + if throttle_dict['count'] >= max_count: + return (throttle_dict, True) + + throttle_dict['count'] += 1 + return (throttle_dict, False) diff --git a/backend/db/__init__.py b/backend/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/db/accounting.py b/backend/db/accounting.py new file mode 100644 index 0000000..c263fb6 --- /dev/null +++ b/backend/db/accounting.py @@ -0,0 +1,586 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Accounting table. Stores aggregated usage metrics. The keys are prefixed +with the type of aggregation and metric stored: +- Per viewpoint: hash_key='vs:' + Aggregate sizes/counts per viewpoint, keyed by the viewpoint + id. Sort keys fall into three categories: + - owned by: 'ow:' only found in default viewpoint. + - shared by: 'sb:' in shared viewpoint, sum of all photos + in episodes owned by 'user_id' + - visible to: 'vt' in shared viewpoint, sum of all photos. not keyed + by user. a given user's "shared with" stats will be 'vt - sb:', + but we do not want to keep per-user shared-by stats. +- Per user: hash_key='us:' + Aggregate sizes/counts per user, keyed by user id. Sort keys are: + - owned by: 'ow' sum of all photos in default viewpoint + - shared by: 'sb' sum of all photos in shared viewpoints and episodes owned by this user + - visible to: 'vt' sum of all photos in shared viewpoint (includes 'sb'). to get the + real count of photos shared with this user but not shared by him, compute 'vt - sb:' +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +from functools import partial +from tornado import gen +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.user import User +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class Accounting(DBRangeObject): + """Accounting object. Stores aggregated information. Currently stores + photo count and sizes per (viewpoint, episode) pair. + """ + + # Maximum op ids to keep. + _MAX_APPLIED_OP_IDS = 10 + + # Types of accounting: each type has its own prefix used to build hash keys. + VIEWPOINT_SIZE = 'vs' + USER_SIZE = 'us' + + # Categories for each type of accounting. Prefix is used to build sort keys. + OWNED_BY = 'ow' + SHARED_BY = 'sb' + VISIBLE_TO = 'vt' + + _table = DBObject._schema.GetTable(vf_schema.ACCOUNTING) + + def __init__(self, hash_key=None, sort_key=None): + """Initialize a new Accounting object.""" + super(Accounting, self).__init__() + self.hash_key = hash_key + self.sort_key = sort_key + self.op_ids = None + self._Reset() + + def _Reset(self): + """Reset counters to 0.""" + self.num_photos = 0 + self.tn_size = 0 + self.med_size = 0 + self.full_size = 0 + self.orig_size = 0 + + def IncrementFromPhotoDict(self, photo_dict): + """Increment counters with the photo stats.""" + self.num_photos += 1 + self.tn_size += photo_dict.get('tn_size', 0) + self.med_size += photo_dict.get('med_size', 0) + self.full_size += photo_dict.get('full_size', 0) + self.orig_size += photo_dict.get('orig_size', 0) + + def IncrementFromPhotoDicts(self, photo_dicts): + """Increment counters with the photo stats.""" + for p in photo_dicts: + self.IncrementFromPhotoDict(p) + + def IncrementFromPhoto(self, photo): + """Increment counters with the photo stats.""" + def _GetOrZero(val): + if val is not None: + return val + else: + return 0 + + self.num_photos += 1 + self.tn_size += _GetOrZero(photo.tn_size) + self.med_size += _GetOrZero(photo.med_size) + self.full_size += _GetOrZero(photo.full_size) + self.orig_size += _GetOrZero(photo.orig_size) + + def IncrementFromPhotos(self, photos): + """Increment counters with the photo stats.""" + for photo in photos: + self.IncrementFromPhoto(photo) + + def DecrementFromPhotoDicts(self, photo_dicts): + """Decrement counters with the photo stats.""" + for p in photo_dicts: + self.num_photos -= 1 + self.tn_size -= p.get('tn_size', 0) + self.med_size -= p.get('med_size', 0) + self.full_size -= p.get('full_size', 0) + self.orig_size -= p.get('orig_size', 0) + + def DecrementFromPhotos(self, photos): + """Decrement counters with the photo stats.""" + def _GetOrZero(val): + if val is not None: + return val + else: + return 0 + for p in photos: + self.num_photos -= 1 + self.tn_size -= _GetOrZero(p.tn_size) + self.med_size -= _GetOrZero(p.med_size) + self.full_size -= _GetOrZero(p.full_size) + self.orig_size -= _GetOrZero(p.orig_size) + + def CopyStatsFrom(self, accounting): + """Copy the usage stats from another accounting object.""" + self.num_photos = accounting.num_photos + self.tn_size = accounting.tn_size + self.med_size = accounting.med_size + self.full_size = accounting.full_size + self.orig_size = accounting.orig_size + + def IncrementStatsFrom(self, accounting): + """Increment stats by another accounting object.""" + self.num_photos += accounting.num_photos + self.tn_size += accounting.tn_size + self.med_size += accounting.med_size + self.full_size += accounting.full_size + self.orig_size += accounting.orig_size + + def DecrementStatsFrom(self, accounting): + """Decrement stats by another accounting object.""" + self.num_photos -= accounting.num_photos + self.tn_size -= accounting.tn_size + self.med_size -= accounting.med_size + self.full_size -= accounting.full_size + self.orig_size -= accounting.orig_size + + def StatsEqual(self, accounting): + """Return true if all stats match those in 'accounting'.""" + return (self.num_photos == accounting.num_photos and + self.tn_size == accounting.tn_size and + self.med_size == accounting.med_size and + self.full_size == accounting.full_size and + self.orig_size == accounting.orig_size) + + def IsZero(self): + return self.StatsEqual(Accounting()) + + def IsOpDuplicate(self, op_id): + """Check whether the 'op_id' is in 'op_id_list_string'. + If it is, return true and leave the original list of op ids untouched. Otherwise, + add the op_id to the list, trim it to a max length of _MAX_APPLIED_OP_IDS = 10 + and return false.""" + ids = self.op_ids.split(',') if self.op_ids is not None else [] + if op_id in ids: + return True + ids.append(op_id) + # Generate a comma-separated string of at most _MAX_APPLIED_OP_IDS elements. + self.op_ids = ','.join(ids[-self._MAX_APPLIED_OP_IDS:]) + return False + + @classmethod + def CreateUserOwnedBy(cls, user_id): + """Create an accounting object (USER_SIZE:, OWNED_BY).""" + return Accounting('%s:%d' % (Accounting.USER_SIZE, user_id), Accounting.OWNED_BY) + + @classmethod + def CreateUserSharedBy(cls, user_id): + """Create an accounting object (USER_SIZE:, SHARED_BY).""" + return Accounting('%s:%d' % (Accounting.USER_SIZE, user_id), Accounting.SHARED_BY) + + @classmethod + def CreateUserVisibleTo(cls, user_id): + """Create an accounting object (USER_SIZE:, VISIBLE_TO).""" + return Accounting('%s:%d' % (Accounting.USER_SIZE, user_id), Accounting.VISIBLE_TO) + + @classmethod + def CreateViewpointOwnedBy(cls, viewpoint_id, user_id): + """Create an accounting object (VIEWPOINT_SIZE:, OWNED_BY:).""" + return Accounting('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint_id), + '%s:%d' % (Accounting.OWNED_BY, user_id)) + + @classmethod + def CreateViewpointSharedBy(cls, viewpoint_id, user_id): + """Create an accounting object (VIEWPOINT_SIZE:, SHARED_BY:).""" + return Accounting('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint_id), + '%s:%d' % (Accounting.SHARED_BY, user_id)) + + @classmethod + def CreateViewpointVisibleTo(cls, viewpoint_id): + """Create an accounting object (VIEWPOINT_SIZE:, VISIBLE_TO).""" + return Accounting('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint_id), + Accounting.VISIBLE_TO) + + @classmethod + def QueryViewpointSharedBy(cls, client, viewpoint_id, user_id, callback, must_exist=True): + """Query for an accounting object (VIEWPOINT_SIZE:, SHARED_BY:).""" + Accounting.Query(client, + Accounting.VIEWPOINT_SIZE + ':' + viewpoint_id, + Accounting.SHARED_BY + ':%d' % user_id, + None, + callback, + must_exist=must_exist) + + @classmethod + def QueryViewpointVisibleTo(cls, client, viewpoint_id, callback, must_exist=True): + """Query for an accounting object (VIEWPOINT_SIZE:, VISIBLE_TO).""" + Accounting.Query(client, + Accounting.VIEWPOINT_SIZE + ':' + viewpoint_id, + Accounting.VISIBLE_TO, + None, + callback, + must_exist=must_exist) + + @classmethod + @gen.coroutine + def QueryUserAccounting(cls, client, user_id): + """Query a single user's accounting entries. Returns an array of [owned_by, shared_by, visible_to] accounting + entries, any of which may be None (eg: if data was not properly populated). + """ + user_hash = '%s:%d' % (Accounting.USER_SIZE, user_id) + result = yield [gen.Task(Accounting.Query, client, user_hash, Accounting.OWNED_BY, None, must_exist=False), + gen.Task(Accounting.Query, client, user_hash, Accounting.SHARED_BY, None, must_exist=False), + gen.Task(Accounting.Query, client, user_hash, Accounting.VISIBLE_TO, None, must_exist=False)] + raise gen.Return(result) + + @classmethod + def ApplyAccounting(cls, client, accounting, callback): + """Apply an accounting object. This involves a query to fetch stats and applied op ids, + check that this operation has not been applied, increment of values and Update. + """ + op_id = Operation.GetCurrent().operation_id + assert op_id is not None, 'accounting update outside an operation' + + def _OnException(accounting, type, value, traceback): + # Entry was modified between Query and Update. Rerun the entire method. + Accounting.ApplyAccounting(client, accounting, callback) + + def _OnQueryAccounting(entry): + if entry is None: + # No previous entry. Set op_id and set replace to false. + # We can submit the accounting object directly since it has not been mutated. + accounting.op_ids = op_id + with util.Barrier(callback, on_exception=partial(_OnException, accounting)) as b: + accounting.Update(client, b.Callback(), replace=False) + else: + prev_op_ids = entry.op_ids + + # Checks whether the op id has been applied and modifies op_ids accordingly. + found = entry.IsOpDuplicate(op_id) + if found: + # This operation has been applied: skip. + callback() + return + + entry.IncrementStatsFrom(accounting) + + # Entry exists: modify the object returned by Query and require that the op_ids + # field has not changed since. If the existing entry was created by dbchk, it will + # not have a op_ids field. Setting expected={'op_ids': None} is not equivalent to + # expected={'op_ids': False}. + with util.Barrier(callback, on_exception=partial(_OnException, accounting)) as b: + entry.Update(client, b.Callback(), expected={'op_ids': prev_op_ids or False}) + + Accounting.Query(client, accounting.hash_key, accounting.sort_key, None, _OnQueryAccounting, must_exist=False) + + +class AccountingAccumulator(object): + """Facilitates collection and application of accounting deltas. + + Typical usage involves calling into an Accounting method one or more times with one of these + accumulators and finally calling the Apply method to apply all of the accounting deltas. + """ + def __init__(self): + """Initializes new AccountingAccumulator object.""" + self.vp_ow_acc_dict = {} + self.vp_vt_acc_dict = {} + self.vp_sb_acc_dict = {} + self.us_ow_acc_dict = {} + self.us_vt_acc_dict = {} + self.us_sb_acc_dict = {} + + def GetViewpointOwnedBy(self, viewpoint_id, user_id): + """Returns the viewpoint owned_by accounting object for the given viewpoint and user.""" + key = (viewpoint_id, user_id) + if key not in self.vp_ow_acc_dict: + self.vp_ow_acc_dict[key] = Accounting.CreateViewpointOwnedBy(viewpoint_id, user_id) + return self.vp_ow_acc_dict[key] + + def GetViewpointVisibleTo(self, viewpoint_id): + """Returns the viewpoint visible_to accounting for the given viewpoint.""" + key = viewpoint_id + if key not in self.vp_vt_acc_dict: + self.vp_vt_acc_dict[key] = Accounting.CreateViewpointVisibleTo(viewpoint_id) + return self.vp_vt_acc_dict[key] + + def GetViewpointSharedBy(self, viewpoint_id, user_id): + """Returns the viewpoint shared_by accounting object for the given viewpoint and user.""" + key = (viewpoint_id, user_id) + if key not in self.vp_sb_acc_dict: + self.vp_sb_acc_dict[key] = Accounting.CreateViewpointSharedBy(viewpoint_id, user_id) + return self.vp_sb_acc_dict[key] + + def GetUserOwnedBy(self, user_id): + """Returns the user owned_by accounting for the given user.""" + key = user_id + if key not in self.us_ow_acc_dict: + self.us_ow_acc_dict[key] = Accounting.CreateUserOwnedBy(user_id) + return self.us_ow_acc_dict[key] + + def GetUserVisibleTo(self, user_id): + """Returns the user visible_to accounting for the given user.""" + key = user_id + if key not in self.us_vt_acc_dict: + self.us_vt_acc_dict[key] = Accounting.CreateUserVisibleTo(user_id) + return self.us_vt_acc_dict[key] + + def GetUserSharedBy(self, user_id): + """Returns the user shared_by accounting for the given user.""" + key = user_id + if key not in self.us_sb_acc_dict: + self.us_sb_acc_dict[key] = Accounting.CreateUserSharedBy(user_id) + return self.us_sb_acc_dict[key] + + @gen.coroutine + def AddFollowers(self, client, viewpoint_id, new_follower_ids): + """Add accounting changes caused by adding followers to a viewpoint. Each follower + user has VISIBLE_TO incremented by the size of the viewpoint VISIBLE_TO. + """ + if len(new_follower_ids) > 0: + # Query the viewpoint visible_to accounting. New followers' visible_to will be adjusted by this much. + vp_vt_acc = yield gen.Task(Accounting.QueryViewpointVisibleTo, client, viewpoint_id, must_exist=False) + + if vp_vt_acc is not None: + # If the viewpoint has data, apply it to the new followers. + for follower_id in new_follower_ids: + self.GetUserVisibleTo(follower_id).CopyStatsFrom(vp_vt_acc) + + @gen.coroutine + def MergeAccounts(self, client, viewpoint_id, target_user_id): + """Add accounting changes caused by adding the given user as a follower of the viewpoint as + part of a merge accounts operation. Increments the target user's VISIBLE_TO by the size of the + viewpoint VISIBLE_TO. + """ + # Query the viewpoint visible_to accounting. The target user's visible_to will be adjusted by this much. + vp_vt_acc = yield gen.Task(Accounting.QueryViewpointVisibleTo, client, viewpoint_id, must_exist=False) + if vp_vt_acc is not None: + self.GetUserVisibleTo(target_user_id).IncrementStatsFrom(vp_vt_acc) + + @gen.coroutine + def RemovePhotos(self, client, user_id, viewpoint_id, photo_ids): + """Add accounting changes caused by removing photos from a user's default viewpoint. + - photo_ids: list of photos that were removed (caller should exclude the ids of any + photos that were already removed). + + We need to query all photos for size information. Creates the following entries: + - (vs:, ow:): stats for user in default viewpoint. + - (us:, ow): overall stats for user. + """ + photo_keys = [DBKey(photo_id, None) for photo_id in photo_ids] + photos = yield gen.Task(Photo.BatchQuery, client, photo_keys, None) + + # Decrement owned by stats on both the viewpoint and the user. + self.GetViewpointOwnedBy(viewpoint_id, user_id).DecrementFromPhotos(photos) + # Don't recompute owned-by stats, just copy them from the viewpoint accounting object. + self.GetUserOwnedBy(user_id).CopyStatsFrom(self.GetViewpointOwnedBy(viewpoint_id, user_id)) + + @gen.coroutine + def RemoveViewpoint(self, client, user_id, viewpoint_id): + """Generate and update accounting entries for a RemoveViewpoint event. + The user will never be removed from their default viewpoint. + + This won't modify the viewpoint stats, but we will query them to determine how much to modify the user stats. + Query: + - (vs:, vt) + - (vs:, sb:) + Creates the following entries: + - (us:, vt): decrement by (vs:, vt). + - (us:, sb): decrement by (vs:, sb:). + """ + # Query the current visible_to and shared_by for the given user and viewpoint. + vp_vt, vp_sb = yield [gen.Task(Accounting.QueryViewpointVisibleTo, client, viewpoint_id, must_exist=False), + gen.Task(Accounting.QueryViewpointSharedBy, client, viewpoint_id, user_id, must_exist=False)] + + # Decrease the associated user consumption by amounts that the user has associated with the viewpoint. + if vp_vt is not None: + self.GetUserVisibleTo(user_id).DecrementStatsFrom(vp_vt) + if vp_sb is not None: + self.GetUserSharedBy(user_id).DecrementStatsFrom(vp_sb) + + @gen.coroutine + def ReviveFollowers(self, client, viewpoint_id, revive_follower_ids): + """Add accounting changes caused by the revival of the given followers. These followers + had removed the viewpoint (which freed up quota), but now have access to it again. Each + follower has VISIBLE_TO incremented by the size of the viewpoint VISIBLE_TO, and SHARED_BY + incremented by the size of the corresponding viewpoint SHARED_BY. + """ + if len(revive_follower_ids) > 0: + # The VISIBLE_TO adjustment is identical to that done for the AddFollowers operation. + yield self.AddFollowers(client, viewpoint_id, revive_follower_ids) + + # Now make the SHARED_BY adjustment. + vp_sb_acc_list = yield [gen.Task(Accounting.QueryViewpointSharedBy, + client, + viewpoint_id, + follower_id, + must_exist=False) + for follower_id in revive_follower_ids] + + for follower_id, vp_sb_acc in zip(revive_follower_ids, vp_sb_acc_list): + if vp_sb_acc is not None: + self.GetUserSharedBy(follower_id).IncrementStatsFrom(vp_sb_acc) + + @gen.coroutine + def SavePhotos(self, client, user_id, viewpoint_id, photo_ids): + """Generate and update accounting entries for a SavePhotos event. + - photo_ids: list of *new* photos that were added (caller should exclude the ids of any + photos that already existed). + + We need to query all photos for size information. Creates the following entries: + - (vs:, ow:): stats for user in default viewpoint. + - (us:, ow): overall stats for user. + """ + photo_keys = [DBKey(photo_id, None) for photo_id in photo_ids] + photos = yield gen.Task(Photo.BatchQuery, client, photo_keys, None) + + # Increment owned by stats on both the viewpoint and the user. + self.GetViewpointOwnedBy(viewpoint_id, user_id).IncrementFromPhotos(photos) + + # Don't recompute owned-by stats, just copy them from the viewpoint accounting object. + self.GetUserOwnedBy(user_id).CopyStatsFrom(self.GetViewpointOwnedBy(viewpoint_id, user_id)) + + @gen.coroutine + def SharePhotos(self, client, sharer_id, viewpoint_id, photo_ids, follower_ids): + """Generate and update accounting entries for a ShareNew or ShareExisting event. + - photo_ids: list of *new* photos that were added (caller should exclude the ids of any + photos that already existed). + - follower_ids: list of ids of all followers of the viewpoint, *including* the sharer + if it is not removed from the viewpoint. + + We need to query all photos for size information. Creates the following entries: + - (vs:, sb:): sum across all new photos for the sharer + - (vs:, vt): sum across all new photos + - (us:, sb): sum across all new photos for the sharer + """ + photo_keys = [DBKey(photo_id, None) for photo_id in photo_ids] + photos = yield gen.Task(Photo.BatchQuery, client, photo_keys, None) + + acc = Accounting() + acc.IncrementFromPhotos(photos) + + # Viewpoint visible_to for viewpoint. + self.GetViewpointVisibleTo(viewpoint_id).IncrementStatsFrom(acc) + + if sharer_id in follower_ids: + # Viewpoint shared_by for sharer. + self.GetViewpointSharedBy(viewpoint_id, sharer_id).IncrementStatsFrom(acc) + + # User shared_by for sharer. + self.GetUserSharedBy(sharer_id).IncrementStatsFrom(acc) + + # Viewpoint visible_to for followers. + for follower_id in follower_ids: + self.GetUserVisibleTo(follower_id).IncrementStatsFrom(acc) + + @gen.coroutine + def Unshare(self, client, viewpoint, ep_dicts, followers): + """Generate and update accounting entries for an Unshare event. Multiple episodes may be + impacted and multiple photos per episode. + - viewpoint: viewpoint that contains the episodes and photos in ep_dicts. + - ep_dicts: dict containing episode and photos ids: {ep_id0: [ph_id0, ph_id1], ep_id1: [ph_id2]}. + - followers: list of all followers of the viewpoint, *including* the sharer. + + We need to look up all photos to fetch size information. Creates the following entries: + - (vs:, sb:): stats for episode owners in this viewpoint. + - (vs:, vt): shared-with stats. + + Creates/adjusts entries for user_accountings based on adjustments to this viewpoint: + - (us:, sb): stats for episode owners. + - (us:, vt): stats for all followers of the viewpoint. + """ + from viewfinder.backend.db.episode import Episode + + # Gather db keys for all episodes and photos. + episode_keys = [] + photo_keys = [] + for episode_id, photo_ids in ep_dicts.iteritems(): + episode_keys.append(DBKey(episode_id, None)) + for photo_id in photo_ids: + photo_keys.append(DBKey(photo_id, None)) + + # Query for all episodes and photos in parallel and in batches. + episodes, photos = yield [gen.Task(Episode.BatchQuery, + client, + episode_keys, + None, + must_exist=False), + gen.Task(Photo.BatchQuery, + client, + photo_keys, + None, + must_exist=False)] + + viewpoint_id = viewpoint.viewpoint_id + user_id = viewpoint.user_id + ep_iter = iter(episodes) + ph_iter = iter(photos) + for episode_id, photo_ids in ep_dicts.iteritems(): + unshare_episode = next(ep_iter) + acc = Accounting() + unshare_photos = [] + for photo_id in photo_ids: + acc.IncrementFromPhoto(next(ph_iter)) + + if viewpoint.IsDefault(): + # Decrement owned by stats on both the viewpoint and the user. + self.GetViewpointOwnedBy(viewpoint_id, user_id).DecrementStatsFrom(acc) + # Don't recompute owned-by stats, just copy them from the viewpoint accounting object. + self.GetUserOwnedBy(user_id).CopyStatsFrom(self.GetViewpointOwnedBy(viewpoint_id, user_id)) + else: + # Viewpoint shared_by for sharer. + self.GetViewpointSharedBy(viewpoint_id, unshare_episode.user_id).DecrementStatsFrom(acc) + # Viewpoint visible_to for viewpoint. + self.GetViewpointVisibleTo(viewpoint_id).DecrementStatsFrom(acc) + # User shared_by for sharer. + self.GetUserSharedBy(unshare_episode.user_id).DecrementStatsFrom(acc) + + # Viewpoint visible_to for followers. + for follower in followers: + if not follower.IsRemoved(): + self.GetUserVisibleTo(follower.user_id).DecrementStatsFrom(acc) + + @gen.coroutine + def UploadEpisode(self, client, user_id, viewpoint_id, ph_dicts): + """Generate and update accounting entries for an UploadEpisode event. + - ph_dicts: list of *new* photo dicts that were added (caller should exclude any photos + that already existed). + + Creates the following entries: + - (vs:, ow:): stats for user in default viewpoint. + - (us:, ow): overall stats for user. + """ + # Increment owned by stats on both the viewpoint and the user. + self.GetViewpointOwnedBy(viewpoint_id, user_id).IncrementFromPhotoDicts(ph_dicts) + # Don't recompute owned-by stats, just copy them from the viewpoint accounting object. + self.GetUserOwnedBy(user_id).CopyStatsFrom(self.GetViewpointOwnedBy(viewpoint_id, user_id)) + + @gen.coroutine + def Apply(self, client): + """Applies all of the accounting deltas that have been collected in the accumulator.""" + # Apply all of the collected user accounting entries. + tasks = [] + for us_ow_acc in self.us_ow_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, us_ow_acc)) + for us_vt_acc in self.us_vt_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, us_vt_acc)) + for us_sb_acc in self.us_sb_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, us_sb_acc)) + yield tasks + + # NOTE: It's important (for idempotency) to complete all user accounting updates before + # starting the viewpoint accounting updates because the removed follower deltas + # are derived from current values of the viewpoint accounting. + + # Apply all of the collected viewpoint accounting entries. + tasks = [] + for vp_ow_acc in self.vp_ow_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, vp_ow_acc)) + for vp_vt_acc in self.vp_vt_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, vp_vt_acc)) + for vp_sb_acc in self.vp_sb_acc_dict.values(): + tasks.append(gen.Task(Accounting.ApplyAccounting, client, vp_sb_acc)) + yield tasks diff --git a/backend/db/activity.py b/backend/db/activity.py new file mode 100644 index 0000000..ece509d --- /dev/null +++ b/backend/db/activity.py @@ -0,0 +1,319 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder activity. + + Activities are associated with a viewpoint and contain a record of all high-level operations + which have modified the structure of the viewpoint in some way. For example, each upload and + share operation will create an activity, since each action creates a new episode within a + viewpoint. Each activity is associated with a custom set of arguments, which are typically + the identifiers of assets involved in the operation. For example, a share activity would + contain the identifiers of the episodes and photos that were shared. See the header for + notification.py for a discussion of how activities are different from notifications. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import json + +from tornado import gen +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.asset_id import IdPrefix, ConstructTimestampAssetId, DeconstructTimestampAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class Activity(DBRangeObject): + """Viewfinder activity data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.ACTIVITY) + + def MakeMetadataDict(self): + """Constructs a dictionary containing activity metadata in a format that conforms to + ACTIVITY in json_schema.py. + """ + activity_dict = self._asdict() + activity_dict[activity_dict.pop('name')] = json.loads(activity_dict.pop('json')) + return activity_dict + + @classmethod + def ConstructActivityId(cls, timestamp, device_id, uniquifier): + """Returns an activity id constructed from component parts. Activities sort from newest + to oldest. See "ConstructTimestampAssetId" for details of the encoding. + """ + return ConstructTimestampAssetId(IdPrefix.Activity, timestamp, device_id, uniquifier) + + @classmethod + def ConstructActivityIdFromOperationId(cls, timestamp, operation_id): + """Returns an activity id constructed by combining the specified timestamp and the + device_id and device_op_id from the operation_id. + """ + device_id, uniquifier = Operation.DeconstructOperationId(operation_id) + return Activity.ConstructActivityId(timestamp, device_id, uniquifier) + + @classmethod + def DeconstructActivityId(cls, activity_id): + """Returns the components of an activity id: timestamp, device_id, and uniquifier.""" + return DeconstructTimestampAssetId(IdPrefix.Activity, activity_id) + + @classmethod + @gen.coroutine + def VerifyActivityId(cls, client, user_id, device_id, activity_id): + """Ensures that a client-provided activity id is valid according to the rules specified + in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Activity, activity_id, has_timestamp=True) + + @classmethod + @gen.coroutine + def CreateAddFollowers(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, follower_ids): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + an "add_followers" operation. + """ + args_dict = {'follower_ids': follower_ids} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'add_followers', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateMergeAccounts(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, target_user_id, source_user_id): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "merge_accounts" operation. + """ + args_dict = {'target_user_id': target_user_id, + 'source_user_id': source_user_id} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'merge_accounts', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreatePostComment(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, cm_dict): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "post_comment" operation. + """ + args_dict = {'comment_id': cm_dict['comment_id']} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'post_comment', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateRemoveFollowers(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, follower_ids): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + an "remove_followers" operation. + """ + args_dict = {'follower_ids': follower_ids} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'remove_followers', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateRemovePhotos(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dicts): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "remove_photos" operation. + """ + args_dict = {'episodes': [{'episode_id': ep_dict['new_episode_id'], + 'photo_ids': ep_dict['photo_ids']} + for ep_dict in ep_dicts]} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'remove_photos', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateSavePhotos(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dicts): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "save_photos" operation. + """ + args_dict = {'episodes': [{'episode_id': ep_dict['new_episode_id'], + 'photo_ids': ep_dict['photo_ids']} + for ep_dict in ep_dicts]} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'save_photos', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateShareExisting(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dicts): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "share_existing" operation. + """ + args_dict = {'episodes': [{'episode_id': ep_dict['new_episode_id'], + 'photo_ids': ep_dict['photo_ids']} + for ep_dict in ep_dicts]} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'share_existing', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateShareNew(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dicts, follower_ids): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "share_new" operation. + """ + args_dict = {'episodes': [{'episode_id': ep_dict['new_episode_id'], + 'photo_ids': ep_dict['photo_ids']} + for ep_dict in ep_dicts], + 'follower_ids': follower_ids} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'share_new', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateUnshare(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dicts): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + a "unshare" operation. + """ + args_dict = {'episodes': [{'episode_id': ep_dict['episode_id'], + 'photo_ids': ep_dict['photo_ids']} + for ep_dict in ep_dicts]} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'unshare', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateUpdateEpisode(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dict): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + an "update_episode" operation. + """ + args_dict = {'episode_id': ep_dict['episode_id']} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'update_episode', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateUpdateViewpoint(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, prev_values): + """Creates an activity that tracks the changes to the specified viewpoint resulting from + an "update_viewpoint" operation. + """ + args_dict = {'viewpoint_id': viewpoint_id} + args_dict.update(prev_values) + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'update_viewpoint', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def CreateUploadEpisode(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, ep_dict, ph_dicts): + """Create an activity that tracks the changes to the specified viewpoint resulting from + a "upload_episode" operation. + """ + args_dict = {'episode_id': ep_dict['episode_id'], + 'photo_ids': [ph_dict['photo_id'] for ph_dict in ph_dicts]} + activity = yield Activity._CreateActivity(client, + user_id, + viewpoint_id, + activity_id, + timestamp, + update_seq, + 'upload_episode', + args_dict) + raise gen.Return(activity) + + @classmethod + @gen.coroutine + def _CreateActivity(cls, client, user_id, viewpoint_id, activity_id, timestamp, + update_seq, name, args_dict): + """Helper method that creates an activity for any kind of operation.""" + activity = yield gen.Task(Activity.Query, client, viewpoint_id, activity_id, None, must_exist=False) + + # If activity doesn't exist, then create it. Otherwise, this is idempotent create case, so just verify user_id. + if activity is None: + from viewfinder.backend.base import message + args_dict['headers'] = dict(version=message.MAX_MESSAGE_VERSION) + + activity = Activity.CreateFromKeywords(viewpoint_id=viewpoint_id, activity_id=activity_id, + user_id=user_id, timestamp=timestamp, update_seq=update_seq, + name=name, json=json.dumps(args_dict)) + yield gen.Task(activity.Update, client) + else: + # Idempotent create. + assert activity.user_id == user_id, (activity, user_id) + + raise gen.Return(activity) diff --git a/backend/db/admin_permissions.py b/backend/db/admin_permissions.py new file mode 100644 index 0000000..a5e83a4 --- /dev/null +++ b/backend/db/admin_permissions.py @@ -0,0 +1,51 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Admin permissions table. + +Stores users allowed to access the admin page. User names should be viewfinder.co user. +The set of rights determines what each user is allowed to access on viewfinder.co/admin. + + 'root': admin functions: DB, logs, counter, etc... + 'support': read-only support function. eg: lookup user id from email address. + +This table only details permissions. Authentication uses per-domain secrets. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject + +@DBObject.map_table_attributes +class AdminPermissions(DBHashObject): + """AdminPermissions object.""" + __slots__ = [] + + # Types of rights. Any number can be set. + ROOT = 'root' + SUPPORT = 'support' + + _table = DBObject._schema.GetTable(vf_schema.ADMIN_PERMISSIONS) + + def __init__(self, username=None, rights=None): + """Initialize a new permissions object.""" + super(AdminPermissions, self).__init__() + self.username = username + if rights is not None: + self.SetRights(rights) + + def IsRoot(self): + """Returns true if 'root' is in the set of rights.""" + return AdminPermissions.ROOT in self.rights + + def IsSupport(self): + """Returns true if 'support' is in the set of rights.""" + return AdminPermissions.SUPPORT in self.rights + + def SetRights(self, rights): + """Clear current set of rights and add the passed-in ones.""" + self.rights = set() + for r in rights: + assert r == self.ROOT or r == self.SUPPORT, 'unknown right: %s' % r + self.rights.add(r) diff --git a/backend/db/analytics.py b/backend/db/analytics.py new file mode 100644 index 0000000..07985dc --- /dev/null +++ b/backend/db/analytics.py @@ -0,0 +1,73 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Analytics table. + +Each row is composed of: +- hash key: entity: : (eg: us:112 for user_id 112) +- range key: base64 timestamp + type +- column 'type': string describing the entry +- column 'payload': optional payload. format based on the type of entry. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import json +import logging +import os +import time + +from tornado import gen +from viewfinder.backend.base import constants, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + +class Analytics(DBRangeObject): + + # Type strings. They should consist of: .. eg: User.CreateProspective + + # Payload for prospective: + # - "register" for prospective user created inline with registration. + # - "share_new=id" with user_id of the user starting the conversation. + # - "add_followed=id" with user_id of the user performing the add_follower op. + USER_CREATE_PROSPECTIVE = 'User.CreateProspective' + + USER_REGISTER = 'User.Register' + + # Payload for terminate: + # - "terminate" on account termination. + # - "merge=id" when merging accounts. id is the target user. + USER_TERMINATE = 'User.Terminate' + + _table = DBObject._schema.GetTable(vf_schema.ANALYTICS) + + def __init__(self): + super(Analytics, self).__init__() + + @classmethod + def CreateSortKey(cls, timestamp, entry_type): + """Create value for sort_key attribute. This is derived from timestamp and type.""" + prefix = util.CreateSortKeyPrefix(timestamp, randomness=False) + return prefix + entry_type + + @classmethod + def Create(cls, **analytics_dict): + """Create a new analytics object with fields from 'analytics_dict'. Sets timestamp if not + specified. Payload may be empty. + """ + create_dict = analytics_dict + if 'timestamp' not in create_dict: + create_dict['timestamp'] = util.GetCurrentTimestamp() + + entity = create_dict['entity'] + entry_type = create_dict['type'] + if entry_type.startswith('User.'): + assert entity.startswith('us:'), 'Wrong entity string for type User.*: %r' % create_dict + + # Always store as int, floats cause problems as sort keys. + create_dict['timestamp'] = int(create_dict['timestamp']) + create_dict['sort_key'] = Analytics.CreateSortKey(create_dict['timestamp'], create_dict['type']) + + return cls.CreateFromKeywords(**create_dict) diff --git a/backend/db/asset_id.py b/backend/db/asset_id.py new file mode 100644 index 0000000..5402981 --- /dev/null +++ b/backend/db/asset_id.py @@ -0,0 +1,218 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder asset id prefixes and helpers. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import struct + +from collections import namedtuple +from tornado import gen +from viewfinder.backend.base import base64hex, util +from viewfinder.backend.base.exceptions import InvalidRequestError, PermissionError +from viewfinder.backend.db.device import Device + + +class IdPrefix(object): + """An asset-id is base-64 hex encoded and then prefixed with a single + char that identifies the type of id. The prefix must be unique and + listed below. + """ + Activity = 'a' + Comment = 'c' + Episode = 'e' + Operation = 'o' + Photo = 'p' + Post = 't' + Viewpoint = 'v' + + @staticmethod + def GetAssetName(prefix): + """Return name of the asset that has the specified prefix.""" + if not hasattr(IdPrefix, '_prefixes'): + IdPrefix._prefixes = {} + for slot in dir(IdPrefix): + if not slot.startswith('_'): + prefix = getattr(IdPrefix, slot) + if isinstance(prefix, str): + assert prefix not in IdPrefix._prefixes + IdPrefix._prefixes[prefix] = slot + + return IdPrefix._prefixes.get(prefix, None) + + @staticmethod + def IsValid(prefix): + """Return true if "prefix" is a uniquely defined id prefix.""" + return IdPrefix.GetAssetName(prefix) is not None + + +AssetIdUniquifier = namedtuple('AssetIdUniquifier', ['client_id', 'server_id']) +"""All asset ids must be globally unique, even across viewpoints or users. +All asset ids contain a device id and a unique numeric id generated by +that device (i.e. the client_id). If this is enough to guarantee +uniqueness, then the "server_id" field will be None. However, if the +server generates an asset, it may need to specify a server-derived byte +string id (i.e. the server_id) in order to provide the required uniqueness. +""" + + +def ConstructAssetId(id_prefix, device_id, uniquifier): + """Constructs an asset id that does not have a timestamp part. The + asset id is base-64 hex encoded so that it sorts the same way as + its binary representation and can be safely included in URLs. The + "id_prefix" is appended to the resulting id. The binary format of + the asset id is as follows: + + device_id (var-length numeric): id of the generating device + client_id (var-length numeric): unique id generated by the device + server_id (byte str): optionally generated by the server + """ + assert IdPrefix.IsValid(id_prefix), id_prefix + + # Encode the device_id. + byte_str = util.EncodeVarLengthNumber(device_id) + + # Append the encoded asset-id uniquifier. + byte_str += _EncodeUniquifier(uniquifier) + + # Base64-hex encode the bytes to preserve ordering while attaining URL-inclusion safety. + return id_prefix + base64hex.B64HexEncode(byte_str, padding=False) + + +def ConstructTimestampAssetId(id_prefix, timestamp, device_id, uniquifier, reverse_ts=True): + """Constructs an asset id that has a leading 4-byte encoded timestamp, + which may be reversed. The asset id is base-64 hex encoded so that it + sorts the same way as its binary representation and can be safely + included in URLs. The "id_prefix" is appended to the resulting id. + The binary format of the asset id is as follows: + + timestamp (32 bits): whole seconds since Unix epoch + device_id (var-length numeric): id of the generating device + client_id (var-length numeric): unique id generated by the device + server_id (byte str): optionally generated by the server + """ + assert IdPrefix.IsValid(id_prefix), id_prefix + + # Drop fractional seconds and possibly reverse the timestamp before converting to raw bytes. + assert timestamp < 1L << 32, timestamp + if reverse_ts: + timestamp = (1L << 32) - int(timestamp) - 1 + byte_str = struct.pack('>I', timestamp) + assert len(byte_str) == 4, timestamp + + # Append the encoded device_id. + byte_str += util.EncodeVarLengthNumber(device_id) + + # Append the encoded asset-id uniquifier. + byte_str += _EncodeUniquifier(uniquifier) + + # Base64-hex encode the bytes for URL-inclusion safety. + return id_prefix + base64hex.B64HexEncode(byte_str, padding=False) + + +def DeconstructAssetId(id_prefix, asset_id): + """Deconstructs an asset id that was previously constructed according + to the rules of "ConstructAssetId" (i.e. no timestamp). Returns a tuple: + (device_id, uniquifier) + """ + assert IdPrefix.IsValid(id_prefix), id_prefix + assert asset_id[0] == id_prefix, asset_id + + # Decode the bytes, which must be base-64 hex encoded. + byte_str = base64hex.B64HexDecode(asset_id[1:], padding=False) + + # Decode the device_id and the uniquifier. + device_id, num_bytes = util.DecodeVarLengthNumber(byte_str) + uniquifier = _DecodeUniquifier(byte_str[num_bytes:]) + + # Return all parts as a tuple. + return device_id, uniquifier + + +def DeconstructTimestampAssetId(id_prefix, asset_id, reverse_ts=True): + """Deconstructs an asset id that was previously constructed according + to the rules of "ConstructTimestampAssetId" (i.e. includes timestamp). + Returns a tuple: + (timestamp, device_id, uniquifier) + """ + assert IdPrefix.IsValid(id_prefix), id_prefix + assert asset_id[0] == id_prefix, asset_id + + # Decode the bytes, which must be base-64 hex encoded. + byte_str = base64hex.B64HexDecode(asset_id[1:], padding=False) + + # Decode the 4-byte timestamp and reverse it if requested. + timestamp, = struct.unpack('>I', byte_str[:4]) + if reverse_ts: + timestamp = (1L << 32) - timestamp - 1 + + # Decode the device_id and the uniquifier. + device_id, num_bytes = util.DecodeVarLengthNumber(byte_str[4:]) + uniquifier = _DecodeUniquifier(byte_str[4 + num_bytes:]) + + # Return all parts as a tuple. + return timestamp, device_id, uniquifier + + +@gen.coroutine +def VerifyAssetId(client, user_id, device_id, prefix_id, asset_id, has_timestamp): + """Verifies that "asset_id" conforms to the following requirements: + 1. The asset prefix must match "prefix_id" and the asset id's format must be valid. + 2. The embedded device_id must match "device_id", or must match another device owned by + "user_id". A device can only create assets with ids that match itself. + 3. The asset_id's uniquifier cannot include a server_id part. Only the server can create + uniquifiers with this part. + """ + try: + asset_name = IdPrefix.GetAssetName(prefix_id).lower() + if has_timestamp: + truncated_ts, embedded_device_id, uniquifier = DeconstructTimestampAssetId(prefix_id, asset_id) + else: + embedded_device_id, uniquifier = DeconstructAssetId(prefix_id, asset_id) + except: + raise InvalidRequestError('%s id "%s" does not have a valid format.' % + (asset_name, asset_id)) + + if embedded_device_id != device_id: + # Query the database to see if the client owns the embedded device id. + device = yield gen.Task(Device.Query, client, user_id, embedded_device_id, None, must_exist=False) + if device is None: + raise PermissionError('User %d and device %d do not have permission to create %s "%s".' % + (user_id, device_id, asset_name, asset_id)) + + if uniquifier.server_id is not None: + raise PermissionError('Clients do not have permission to create %s "%s".' % + (asset_name, asset_id)) + + +def _EncodeUniquifier(uniquifier): + """If "uniquifier" is an int or long, then assumes that there is no + server_id component needed to make the asset id unique. Otherwise, + expects "uniquifier" to be a tuple containing (client_id, + server_id). + + Encodes the client_id and server_id as a combined byte str and returns + it. + """ + if type(uniquifier) in (int, long): + byte_str = util.EncodeVarLengthNumber(uniquifier) + else: + client_id, server_id = uniquifier + assert server_id is None or type(server_id) in (str, unicode), (server_id, type(server_id)) + + byte_str = util.EncodeVarLengthNumber(client_id) + + if server_id is not None: + byte_str += str(server_id) + + return byte_str + + +def _DecodeUniquifier(byte_str): + """Decodes the byte str produced by "_EncodeUniquifier" and returns + the component parts as an AssetIdUniquifier tuple. + """ + client_id, num_bytes = util.DecodeVarLengthNumber(byte_str) + server_id = byte_str[num_bytes:] if num_bytes < len(byte_str) else None + return AssetIdUniquifier(client_id, server_id) diff --git a/backend/db/async_aws_sts.py b/backend/db/async_aws_sts.py new file mode 100644 index 0000000..6b3eeb6 --- /dev/null +++ b/backend/db/async_aws_sts.py @@ -0,0 +1,107 @@ +#!/bin/env python +# +# Copyright 2012 bit.ly +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Created by Dan Frank on 2012-01-25. +Copyright (c) 2012 bit.ly. All rights reserved. +""" + +import functools +from tornado.httpclient import HTTPRequest +from tornado.httpclient import AsyncHTTPClient +import xml.sax + +import boto +from boto.sts.connection import STSConnection +from boto.sts.credentials import Credentials +from boto.exception import BotoServerError + +class InvalidClientTokenIdError(BotoServerError): + """Error subclass to indicate that the client's token(s) is/are invalid. + """ + pass + +class AsyncAwsSts(STSConnection): + """Class that manages session tokens. Users of AsyncDynamoDB should not + need to worry about what goes on here. + + Usage: Keep an instance of this class (though it should be cheap to + re instantiate) and periodically call get_session_token to get a new + Credentials object when, say, your session token expires. + """ + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/', + converter=None): + STSConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, debug, + https_connection_factory, region, path, converter) + + def get_session_token(self, callback): + """Gets a new Credentials object with a session token, using this + instance's aws keys. Callback should operate on the new Credentials obj, + or else a boto.exception.BotoServerError. + """ + return self.get_object('GetSessionToken', {}, Credentials, verb='POST', callback=callback) + + def get_object(self, action, params, cls, path="/", parent=None, verb="GET", callback=None): + """Get an instance of `cls` using `action`.""" + if not parent: + parent = self + self.make_request(action, params, path, verb, + functools.partial(self._finish_get_object, callback=callback, parent=parent, cls=cls)) + + def _finish_get_object(self, response_body, callback, cls=None, parent=None, error=None): + """Process the body returned by STS. If an error is present, + convert from a tornado error to a boto error. + """ + if error: + if error.code == 403: + error_class = InvalidClientTokenIdError + else: + error_class = BotoServerError + return callback(None, error=error_class(error.code, error.message, response_body)) + obj = cls(parent) + h = boto.handler.XmlHandler(obj, parent) + xml.sax.parseString(response_body, h) + return callback(obj) + + def make_request(self, action, params={}, path='/', verb='GET', callback=None): + """Make an async request. This handles the logic of translating + from boto params to a tornado request obj, issuing the request, + and passing back the body. + + The callback should operate on the body of the response, and take + an optional error argument that will be a tornado error. + """ + request = HTTPRequest('https://%s' % self.host, method=verb) + request.params = params + request.auth_path = '/' # need this for auth + request.host = self.host # need this for auth + if action: + request.params['Action'] = action + if self.APIVersion: + request.params['Version'] = self.APIVersion + self._auth_handler.add_auth(request) # add signature + http_client = AsyncHTTPClient() + http_client.fetch(request, functools.partial(self._finish_make_request, callback=callback)) + + def _finish_make_request(self, response, callback): + if response.error: + return callback(response.body, error=response.error) + return callback(response.body) diff --git a/backend/db/asyncdynamo.py b/backend/db/asyncdynamo.py new file mode 100644 index 0000000..95b3c68 --- /dev/null +++ b/backend/db/asyncdynamo.py @@ -0,0 +1,237 @@ +# Copyright 2012 bit.ly +# Copyright 2012 Viewfinder Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Created by Dan Frank on 2012-01-23. +Copyright (c) 2012 bit.ly. All rights reserved. +""" + +import sys +assert sys.version_info >= (2, 7), "run this with python2.7" + +import functools +import json +import logging +import time + +from async_aws_sts import AsyncAwsSts, InvalidClientTokenIdError +from boto.auth import HmacAuthV3HTTPHandler +from boto.connection import AWSAuthConnection +from boto.exception import DynamoDBResponseError +from boto.provider import Provider +from collections import deque +from tornado import httpclient, ioloop +from tornado.httpclient import HTTPRequest +from viewfinder.backend.base.exceptions import DBProvisioningExceededError, DBLimitExceededError, DBConditionalCheckFailedError + +PENDING_SESSION_TOKEN_UPDATE = "this is not your session token" + + +class AsyncDynamoDB(AWSAuthConnection): + """The main class for asynchronous connections to DynamoDB. + + The user should maintain one instance of this class (though more + than one is ok), parametrized with the user's access key and secret + key. Make calls with make_request or the helper methods, and + AsyncDynamoDB will maintain session tokens in the background. + """ + + DefaultHost = 'dynamodb.us-east-1.amazonaws.com' + """The default DynamoDB API endpoint to connect to.""" + + ServiceName = 'DynamoDB' + """The name of the Service""" + + Version = '20111205' + """DynamoDB API version.""" + + ExpiredSessionError = 'ExpiredTokenException' + """The error response returned when session token has expired""" + + UnrecognizedClientException = 'UnrecognizedClientException' + """Another error response that is possible with a bad session token""" + + ProvisionedThroughputExceededException = 'ProvisionedThroughputExceededException' + """Provisioned throughput for requests to table exceeded.""" + + LimitExceededException = 'LimitExceededException' + """Limit for subscriber requests exceeded.""" + + ConditionalCheckFailedException = 'ConditionalCheckFailedException' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host=None, debug=0, session_token=None, + authenticate_requests=True, validate_cert=True, max_sts_attempts=3): + if not host: + host = self.DefaultHost + self.validate_cert = validate_cert + self.authenticate_requests = authenticate_requests + AWSAuthConnection.__init__(self, host, + aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + debug=debug, security_token=session_token) + self.pending_requests = deque() + self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key) + assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) + self.max_sts_attempts = max_sts_attempts + + def _init_session_token_cb(self, error=None): + if error: + logging.warn("Unable to get session token: %s" % error) + + def _required_auth_capability(self): + return ['hmac-v3-http'] + + def _update_session_token(self, callback, attempts=0, bypass_lock=False): + """Begins the logic to get a new session token. Performs checks to + ensure that only one request goes out at a time and that backoff + is respected, so it can be called repeatedly with no ill + effects. Set bypass_lock to True to override this behavior. + """ + if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: + return + self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token + return self.sts.get_session_token( + functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) + + def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): + """Callback to use with `async_aws_sts`. The 'provider' arg is a + bit misleading, it is a relic from boto and should probably be + left to its default. This will take the new Credentials obj from + `async_aws_sts.get_session_token()` and use it to update + self.provider, and then will clear the deque of pending requests. + + A callback is optional. If provided, it must be callable without + any arguments. + """ + def raise_error(): + # get out of locked state + self.provider.security_token = None + if callable(callback): + return callback(error=error) + else: + logging.error(error) + raise error + if error: + if isinstance(error, InvalidClientTokenIdError): + # no need to retry if error is due to bad tokens + raise_error() + else: + if attempts > self.max_sts_attempts: + raise_error() + else: + seconds_to_wait = (0.1 * (2 ** attempts)) + logging.warning("Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) + ioloop.IOLoop.current().add_timeout(time.time() + seconds_to_wait, + functools.partial(self._update_session_token, attempts=attempts + 1, callback=callback, bypass_lock=True)) + return + else: + self.provider = Provider(provider, + creds.access_key, + creds.secret_key, + creds.session_token) + # force the correct auth, with the new provider + self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) + while self.pending_requests: + request = self.pending_requests.pop() + request() + if callable(callback): + return callback() + + def make_request(self, action, body='', callback=None, object_hook=None): + """Make an asynchronous HTTP request to DynamoDB. Callback should + operate on the decoded json response (with object hook applied, of + course). It should also accept an error argument, which will be a + boto.exception.DynamoDBResponseError. + + If there is not a valid session token, this method will ensure + that a new one is fetched and cache the request when it is + retrieved. + """ + this_request = functools.partial(self.make_request, action=action, + body=body, callback=callback, object_hook=object_hook) + if self.authenticate_requests and self.provider.security_token in [None, PENDING_SESSION_TOKEN_UPDATE]: + # we will not be able to complete this request because we do not have a valid session token. + # queue it and try to get a new one. _update_session_token will ensure that only one request + # for a session token goes out at a time + self.pending_requests.appendleft(this_request) + def cb_for_update(error=None): + # create a callback to handle errors getting session token + # callback here is assumed to take a json response, and an instance of DynamoDBResponseError + if error: + raise DynamoDBResponseError(error.status, error.reason, body={'message': error.body}) + else: + return + self._update_session_token(cb_for_update) + return + headers = {'X-Amz-Target' : '%s_%s.%s' % (self.ServiceName, self.Version, action), + 'Content-Type' : 'application/x-amz-json-1.0', + 'Content-Length' : str(len(body))} + request = HTTPRequest('https://%s' % self.host, + method='POST', + headers=headers, + body=body, + validate_cert=self.validate_cert) + request.path = '/' # Important! set the path variable for signing by boto. '/' is the path for all dynamodb requests + if self.authenticate_requests: + self._auth_handler.add_auth(request) # add signature to headers of the request + + http_client = httpclient.AsyncHTTPClient() + http_client.fetch(request, functools.partial( + self._finish_make_request, callback=callback, orig_request=this_request, + token_used=self.provider.security_token, object_hook=object_hook)) + + def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): + """Check for errors and decode the json response (in the tornado + response body), then pass on to orig callback. This method also + contains some of the logic to handle reacquiring session tokens. + """ + if not response.body: + assert response.error, 'How can there be no response body and no error? Response: %s' % response + raise DynamoDBResponseError(response.error.code, response.error.message, None) + + json_response = json.loads(response.body, object_hook=object_hook) + if response.error: + aws_error_type = None + try: + # The error code should be in the __type field of the json response, and should be a string + # in the form 'namespace.version#errorcode'. If the field doesn't exist or is in some other form, + # just treat this as an unknown error type. + aws_error_type = json_response.get('__type').split('#')[1] + except: + aws_error_type = None + + if aws_error_type == self.ExpiredSessionError or aws_error_type == self.UnrecognizedClientException: + if self.provider.security_token == token_used: + # The token that we used has expired, wipe it out. + self.provider.security_token = None + + # make_request will handle logic to get a new token if needed, and queue until it is fetched + return orig_request() + + if aws_error_type == AsyncDynamoDB.ProvisionedThroughputExceededException: + raise DBProvisioningExceededError(json_response['message']) + + if aws_error_type == AsyncDynamoDB.LimitExceededException: + raise DBLimitExceededError(json_response['message']) + + if aws_error_type == AsyncDynamoDB.ConditionalCheckFailedException: + raise DBConditionalCheckFailedError(json_response['message']) + + raise DynamoDBResponseError(response.error.code, response.error.message, json_response) + + return callback(json_response) diff --git a/backend/db/base.py b/backend/db/base.py new file mode 100644 index 0000000..58990ed --- /dev/null +++ b/backend/db/base.py @@ -0,0 +1,546 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Base object for building python classes to represent the data +in a database row. + +See DBHashObject and DBRangeObject. + + DBObject: base class of all data objects +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from functools import partial +import logging + +from tornado import gen +from viewfinder.backend.base import util +from tornado.concurrent import return_future +from viewfinder.backend.db import db_client, query_parser, schema, vf_schema +from viewfinder.backend.db.versions import Version + +class DBObject(object): + """Base class for representing a row of data. Setting a column value + to None will delete the column from the datastore on Update(). + """ + _VISIT_LIMIT = 50 + + _schema = vf_schema.SCHEMA + + __slots__ = ['_columns', '_reindex'] + + def __init__(self, columns=None): + """The base datastore object class manages columns according to + the database schema as defined by the subclass' schema table + definition. However, derived classes can override the column set + by specifying the "columns" argument. Columns of type IndexTermsColumn + are ignored here. They will not create column values which can be + accessed via the __{Get,Set}Property() methods. + + Creates a new python property for each column in the table for the + data object according to the schema. This is only done once per class, + as properties actually modify the class, not the instance. + """ + self._columns = {} + self._reindex = False + columns = columns or self._table.GetColumns() + for c in columns: + if not isinstance(c, schema.IndexTermsColumn): + self._columns[c.name] = c.NewInstance() + + @staticmethod + def map_table_attributes(cls): + """Class decorator which adds properties for all columns defined in a table. + + The class must define a class attribute _table. + + Example: + @DBObject.map_table_attributes + class Foo(DBRangeObject): + _table = DBObject._schema.GetTable(vf_schema.FOO) + """ + assert issubclass(cls, DBObject) + for c in cls._table.GetColumns(): + if not isinstance(c, schema.IndexTermsColumn): + fget = (lambda name: lambda self: self.__GetProperty(name))(c.name) + fset = (lambda name: lambda self, value: self.__SetProperty(name, value))(c.name) + setattr(cls, c.name, property(fget, fset)) + return cls + + def __dir__(self): + return self._columns.keys() + + def __repr__(self): + items = [] + for name, column in self._columns.iteritems(): + if column.Get() is None: + continue + value = repr(column.Get()) + if self.ShouldScrubColumn(name): + value = '...scrubbed %s bytes...' % len(value) + items.append((name, value)) + return '{' + ', '.join('\'%s\': %s' % (n, v) for n, v in items) + '}' + + @classmethod + def ShouldScrubColumn(cls, name): + """Override to return True for columns that should not appear in logs.""" + return False + + def __GetProperty(self, name): + return self._columns[name].Get() + + def __SetProperty(self, name, value): + return self._columns[name].Set(value) + + def _asdict(self): + return dict([(n, c.Get(asdict=True)) for n, c in self._columns.items() \ + if c.Get() is not None]) + + def _Clone(self): + # Construct new instance of this type and transfer raw in-memory column values. + o = type(self)() + for n, col in self._columns.items(): + o._columns[n]._value = col._value + return o + + def _IsModified(self, name): + """Returns whether or not a column value has been modified.""" + return self._columns[name].IsModified() + + def GetColNames(self): + """Returns all column names.""" + return self._columns.keys() + + def GetModifiedColNames(self): + """Returns all column names where the column value has been modified.""" + return [c.col_def.name for c in self._columns.values() if c.IsModified()] + + def SetReindexOnUpdate(self, reindex): + """Sets the _reindex boolean. If set to True, index terms for all + columns will be re-generated on update, regardless of whether or + not the column has been modified. This is used during data + migrations when the indexing algorithm for a particular column + type (or types) has been modified. Only generates writes (and + deletes for pre-existing, now obsolete terms) to the index table + when terms for a column change. + """ + self._reindex = reindex + + @classmethod + def CreateFromKeywords(cls, **obj_dict): + """Creates a new object of type 'cls' with attributes as specified + in 'obj_dict'. The key columns must be present in the attribute + dictionary. Returns new object instance. + """ + assert obj_dict.has_key(cls._table.hash_key_col.name) + if cls._table.range_key_col: + assert obj_dict.has_key(cls._table.range_key_col.name), (cls._table.range_key_col.name, obj_dict) + o = cls() + o.UpdateFromKeywords(**obj_dict) + o._columns[schema.Table.VERSION_COLUMN.name].Set(Version.GetCurrentVersion()) + return o + + def UpdateFromKeywords(self, **obj_dict): + """Updates the contents of the object according to **obj_dict.""" + for k, v in obj_dict.items(): + if k in self._columns: + self._columns[k].Set(v) + else: + raise KeyError('column %s (value %r) not found in class %s' % (k, v, self.__class__)) + + def HasMismatchedValues(self, mismatch_allowed_set=None, **obj_dict): + """Check that each of the dictionary values matches what's in the object. + The only keys that don't need to match are ones contained in the mismatch_allowed_set. + Returns: True if mismatch found. Otherwise, False. + """ + for k, v in obj_dict.items(): + if k in self._columns: + if mismatch_allowed_set is None or k not in mismatch_allowed_set: + if self._columns[k].Get(asdict=isinstance(v, dict)) != v: + return True + return False + + @return_future + def Update(self, client, callback, expected=None, replace=True, return_col_names=False): + """Updates or inserts the object. Only modified columns are + updated. Updates the index terms first and finally the object, so + the update operation, on retry, will be idempotent. + + 'expected' are preconditions for attribute values for the update + to succeed. + + If 'replace' is False, forces a conditional update which verifies + that the primary key does not already exist in the datastore. + + If 'return_col_names' is True, 'callback' is invoked with a list + of the modified column names. + """ + mod_cols = [c for c in self._columns.values() if c.IsModified()] + if return_col_names: + callback = partial(callback, [c.col_def.name for c in mod_cols]) + + if not mod_cols and not self._reindex: + callback() + return + + # Transform expected attributes dict to refer to column keys instead of names. + if expected: + expected = dict([(self._table.GetColumn(k).key, v) for k, v in expected.items()]) + else: + expected = {} + + def _OnUpdate(result): + [col.OnUpdate() for col in mod_cols] + callback() + + def _OnUpdateIndexTerms(term_attrs): + attrs = dict() + for c in mod_cols: + update = c.Update() + if update: + attrs[c.col_def.key] = update + if term_attrs: + attrs.update(term_attrs) + if not replace: + expected[self._table.hash_key_col.key] = False + if self._table.range_key_col: + expected[self._table.range_key_col.key] = False + client.UpdateItem(table=self._table.name, key=self.GetKey(), + attributes=attrs, expected=expected, callback=_OnUpdate) + + def _OnQueryIndexTerms(term_updates, result): + old_dict = result.attributes or {} + term_attrs = {} + add_terms = {} # dict of term dicts by term key + del_terms = [] # list of term keys + for name, update in term_updates.items(): + key = self._table.GetColumn(name).key + ':t' + terms = set(update.value.keys()) if update.value else set() + + # Special check here; you cannot 'PUT' an empty set. Must 'DELETE'. + if update.action == 'PUT' and not terms: + term_attrs[key] = db_client.UpdateAttr(value=None, action='DELETE') + else: + term_attrs[key] = db_client.UpdateAttr(value=list(terms), action=update.action) + + # Compute which index terms to add and which to delete. + if update.action == 'PUT': + old_terms = set(old_dict.get(key, [])) + add_terms.update(dict([(t, update.value[t]) for t in terms.difference(old_terms)])) + del_terms += old_terms.difference(terms) + elif update.action == 'ADD': + add_terms.update(update.value) + elif update.action == 'DELETE': + del_terms += terms + + # Add and delete all terms as necessary. + with util.Barrier(partial(_OnUpdateIndexTerms, term_attrs)) as b: + index_key = self._GetIndexKey() + for term, data in add_terms.items(): + attrs = {'d': data} if data else {} + client.PutItem(table=vf_schema.INDEX, callback=b.Callback(), attributes=attrs, + key=db_client.DBKey(hash_key=term, range_key=index_key)) + for term in del_terms: + client.DeleteItem(table=vf_schema.INDEX, callback=b.Callback(), + key=db_client.DBKey(hash_key=term, range_key=index_key)) + + if isinstance(self._table, schema.IndexedTable): + index_cols = mod_cols if not self._reindex else \ + [c for c in self._columns.values() if c.Get() is not None] + index_cols = [c for c in index_cols if c.col_def.indexer] + # Get a dictionary of term updates for the object. + term_updates = dict([(c.col_def.name, c.IndexTerms()) for c in index_cols]) + col_names = [n for n, u in term_updates.items() if u.action == 'PUT'] + # For any term updates which are PUT, fetch the previous term sets. + self._QueryIndexTerms(client, col_names=col_names, + callback=partial(_OnQueryIndexTerms, term_updates)) + else: + _OnUpdateIndexTerms(None) + + def Delete(self, client, callback, expected=None): + """Deletes all columns of the object and all associated index + terms. Deletes the index terms first and finally the object, so + the deletion operation, on retry, will be idempotent. + + 'expected' are preconditions for attribute values for the delete + to succeed. + """ + # Transform expected attributes dict to refer to column keys instead of names. + if expected: + expected = dict([(self._table.GetColumn(k).key, v) for k, v in expected.items()]) + + def _OnDelete(result): + callback() + + def _OnDeleteIndexTerms(): + client.DeleteItem(table=self._table.name, key=self.GetKey(), + callback=_OnDelete, expected=expected) + + def _OnQueryIndexTerms(get_result): + terms = [t for term_set in get_result.attributes.values() for t in term_set] + with util.Barrier(_OnDeleteIndexTerms) as b: + index_key = self._GetIndexKey() + [client.DeleteItem(table=vf_schema.INDEX, + key=db_client.DBKey(hash_key=term, range_key=index_key), + callback=b.Callback()) for term in terms] + + if isinstance(self._table, schema.IndexedTable): + assert expected is None, expected + self._QueryIndexTerms(client, col_names=self._table.GetColumnNames(), + callback=_OnQueryIndexTerms) + else: + _OnDeleteIndexTerms() + + def _QueryIndexTerms(self, client, col_names, callback): + """Queries the index terms for the specified columns. If no + columns are specified, invokes callback immediately. When a column + is indexed, the set of index terms produced is stored near the + column value to be queried on modifications. Having access to the + old set is especially crucial if the indexing algorithm changes. + """ + idx_cols = [self._columns[name] for name in col_names if self._table.GetColumn(name).indexer] + attrs = [c.col_def.key + ':t' for c in idx_cols] + + def _OnQuery(get_result): + """Handle case of new object and a term attributes query failure.""" + if get_result is None: + callback(db_client.GetResult(attributes=dict([(a, set()) for a in attrs]), read_units=0)) + else: + callback(get_result) + + if attrs: + client.GetItem(table=self._table.name, key=self.GetKey(), attributes=attrs, + must_exist=False, consistent_read=True, callback=_OnQuery) + else: + # This may happen if no indexed columns were updated. Simply + # supply an empty attribute dict to the callback. + callback(db_client.GetResult(attributes=dict(), read_units=0)) + + def _GetIndexKey(self): + """Returns the indexing key for this object by calling the + _MakeIndexKey class method, which is overridden by derived classes. + """ + return self._MakeIndexKey(self.GetKey()) + + @classmethod + def _CreateFromQuery(cls, **attr_dict): + """Creates a new instance of cls and sets the values of its + columns from 'attr_dict'. Returns the new object instance. + """ + assert attr_dict.has_key(cls._table.hash_key_col.key), attr_dict + if cls._table.range_key_col: + assert attr_dict.has_key(cls._table.range_key_col.key), attr_dict + + o = cls() + for k, v in attr_dict.items(): + name = cls._table.GetColumnName(k) + o._columns[name].Load(v) + + return o + + @classmethod + def Scan(cls, client, col_names, callback, limit=None, excl_start_key=None, + scan_filter=None): + """Scans the table up to a count of 'limit', starting at the hash + key value provided in 'excl_start_key'. Invokes the callback with + the list of elements and the last scanned key (list, last_key). + The last_key will be None if the last item was scanned. + + 'scan_filter' is a map from attribute name to a tuple of + ([attr_value], ('EQ'|'LE'|'LT'|'GE'|'GT'|'BEGINS_WITH')), + --or-- ([start_attr_value, end_attr_value], 'BETWEEN'). + """ + if limit == 0: + callback(([], None)) + + col_set = cls._CreateColumnSet(col_names) + + # Convert scan filter from attribute names to keys. + if scan_filter: + scan_filter = dict([(cls._table.GetColumn(k).key, v) for k, v in scan_filter.items()]) + + def _OnScan(result): + objs = [] + for item in result.items: + objs.append(cls._CreateFromQuery(**item)) + callback((objs, result.last_key)) + + client.Scan(table=cls._table.name, callback=_OnScan, + attributes=[cls._table.GetColumn(name).key for name in col_set], + limit=limit, excl_start_key=excl_start_key, scan_filter=scan_filter) + + @classmethod + @gen.engine + def BatchQuery(cls, client, keys, col_names, callback, + must_exist=True, consistent_read=False): + """Queries for a batch of items identified by DBKey objects in the 'keys' array. Projects + the specified columns (or all columns if col_names==None). If 'must_exist' is False, then + return None for each item that does not exist in the database. + """ + col_set = cls._CreateColumnSet(col_names) + + request = db_client.BatchGetRequest(keys=keys, + attributes=[cls._table.GetColumn(name).key + for name in col_set], + consistent_read=consistent_read) + result = yield gen.Task(client.BatchGetItem, + batch_dict={cls._table.name: request}, + must_exist=must_exist) + + result_objects = [] + for item in result[cls._table.name].items: + if item is not None: + result_objects.append(cls._CreateFromQuery(**item)) + else: + result_objects.append(None) + + callback(result_objects) + + @classmethod + def KeyQuery(cls, client, key, col_names, callback, + must_exist=True, consistent_read=False): + """Queries the specified columns (or all columns if + col_names==None), using key as the object hash key. + """ + col_set = cls._CreateColumnSet(col_names) + + def _OnQuery(result): + o = None + if result and result.attributes: + o = cls._CreateFromQuery(**result.attributes) + callback(o) + + client.GetItem(table=cls._table.name, key=key, + attributes=[cls._table.GetColumn(name).key for name in col_set], + must_exist=must_exist, consistent_read=consistent_read, + callback=_OnQuery) + + @classmethod + def IndexQueryKeys(cls, client, bound_query_str, callback, + start_index_key=None, end_index_key=None, + limit=50, consistent_read=False): + """Returns a sequence of object keys to 'callback' resulting from + execution of 'bound_query_str'. + """ + def _OnQueryKeys(index_keys): + callback([cls._ParseIndexKey(index_key) for index_key in index_keys]) + + try: + start_key = cls._MakeIndexKey(start_index_key) if start_index_key is not None else None + end_key = cls._MakeIndexKey(end_index_key) if end_index_key is not None else None + query, param_dict = query_parser.CompileQuery(cls._schema, bound_query_str) + query.Evaluate(client, + callback=_OnQueryKeys, + start_key=start_key, + end_key=end_key, + limit=limit, + consistent_read=consistent_read, + param_dict=param_dict) + except: + logging.exception('query evaluates to empty: ' + str(bound_query_str)) + callback([]) + + @classmethod + @gen.engine + def IndexQuery(cls, client, bound_query_str, col_names, callback, + start_index_key=None, end_index_key=None, + limit=50, consistent_read=False): + """Returns a sequence of Objects resulting from the execution of + 'query' as the first parameter to 'callback'. Only the columns + specified in 'col_names' are queried, or all columns if None. + """ + try: + start_key = cls._MakeIndexKey(start_index_key) if start_index_key is not None else None + end_key = cls._MakeIndexKey(end_index_key) if end_index_key is not None else None + query, param_dict = query_parser.CompileQuery(cls._schema, bound_query_str) + index_keys = yield gen.Task(query.Evaluate, + client, + start_key=start_key, + end_key=end_key, + limit=limit, + consistent_read=consistent_read, + param_dict=param_dict) + except: + logging.exception('query evaluates to empty: ' + str(bound_query_str)) + callback([]) + return + + query_keys = [cls._ParseIndexKey(index_key) for index_key in index_keys] + objects = yield gen.Task(cls._GetIndexedObjectClass().BatchQuery, + client, + query_keys, + col_names, + must_exist=False, + consistent_read=consistent_read) + # Compact results + compacted_result = [obj for obj in objects if obj is not None] + callback(compacted_result) + + @classmethod + def VisitIndexKeys(cls, client, bound_query_str, visitor, callback, + start_index_key=None, end_index_key=None, consistent_read=False): + """Query for all object keys in the specified key range. For each key, + invoke the "visitor" function: + + visitor(object_key, visit_callback) + + When the visitor function has completed the visit, it should invoke + "visit_callback" with no parameters. Once all object keys have been + visited, then "callback" is invoked. + """ + def _OnQueryKeys(index_keys): + if len(index_keys) < DBObject._VISIT_LIMIT: + barrier_callback = callback + else: + barrier_callback = partial(DBObject.VisitIndexKeys, client, bound_query_str, visitor, callback, + start_index_key=index_keys[-1], end_index_key=end_index_key, + consistent_read=consistent_read) + + with util.Barrier(barrier_callback) as b: + for index_key in index_keys: + visitor(index_key, callback=b.Callback()) + + cls.IndexQueryKeys(client, bound_query_str, _OnQueryKeys, limit=DBObject._VISIT_LIMIT, + start_index_key=start_index_key, end_index_key=end_index_key, + consistent_read=consistent_read) + + @classmethod + def VisitIndex(cls, client, bound_query_str, visitor, col_names, callback, + start_index_key=None, end_index_key=None, consistent_read=False): + """Query for all objects in the specified key range. For each object, + invoke the "visitor" function: + + visitor(object, visit_callback) + + When the visitor function has completed the visit, it should invoke + "visit_callback" with no parameters. Once all objects have been + visited, then "callback" is invoked. + """ + def _OnQuery(objects): + if len(objects) < DBObject._VISIT_LIMIT: + barrier_callback = callback + else: + barrier_callback = partial(DBObject.VisitIndex, client, bound_query_str, visitor, col_names, callback, + start_index_key=objects[-1]._GetIndexKey(), end_index_key=end_index_key, + consistent_read=consistent_read) + + with util.Barrier(barrier_callback) as b: + for object in objects: + visitor(object, b.Callback()) + + cls.IndexQuery(client, bound_query_str, col_names, _OnQuery, limit=DBObject._VISIT_LIMIT, + start_index_key=start_index_key, end_index_key=end_index_key, + consistent_read=consistent_read) + + @classmethod + def _CreateColumnSet(cls, col_names): + """Creates a set of column names from the 'col_names' list (all columns in the table if + col_names == None). Ensures that the hash key, range key, and version column are always + included in the set. + """ + col_set = set(col_names or cls._table.GetColumnNames()) + col_set.add(cls._table.hash_key_col.name) + if cls._table.range_key_col: + col_set.add(cls._table.range_key_col.name) + col_set.add(schema.Table.VERSION_COLUMN.name) + return col_set diff --git a/backend/db/client_log.py b/backend/db/client_log.py new file mode 100644 index 0000000..b04e54b --- /dev/null +++ b/backend/db/client_log.py @@ -0,0 +1,97 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder storage of client logs. + +Viewfinder client applications can write logs to S3 in a manner +analogous to server operation logs. The client makes an API request to +/service/get_client_log and supplies a unique log identification +number. MD5 and bytes may be optionally specified as well. The +response from the server contains a permissioned S3 PUT URL. + + ClientLog +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import datetime +import logging +import re +import time + +from functools import partial +from viewfinder.backend.base import constants, util +from viewfinder.backend.storage.object_store import ObjectStore + +CLIENT_LOG_CONTENT_TYPE = 'application/octet-stream' +MAX_CLIENT_LOGS = 1000 + +class ClientLog(object): + """Viewfinder client log.""" + @classmethod + def _IsoDate(cls, timestamp): + """Gets an ISO date string for the specified UTC "timestamp".""" + return datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d') + + @classmethod + def _LogKeyPrefix(cls, user_id, iso_date_str): + """Creates a key prefix for user log files based on "user_id" and + the "iso_date_str". + """ + return '%d/%s' % (user_id, iso_date_str) + + @classmethod + def GetPutUrl(cls, user_id, device_id, timestamp, client_log_id, + content_type=CLIENT_LOG_CONTENT_TYPE, + content_md5=None, max_bytes=10 << 20): + """Returns a URL for the client to write device logs to S3. URLs + expire by default in a day and expect content-type + CLIENT_LOG_CONTENT_TYPE. + """ + iso_date_str = ClientLog._IsoDate(timestamp) + key = '%s/dev-%d-%s' % (ClientLog._LogKeyPrefix(user_id, iso_date_str), + device_id, client_log_id) + + obj_store = ObjectStore.GetInstance(ObjectStore.USER_LOG) + return obj_store.GenerateUploadUrl( + key, content_type=content_type, content_md5=content_md5, + expires_in=constants.SECONDS_PER_DAY, max_bytes=max_bytes) + + @classmethod + def ListClientLogs(cls, user_id, start_timestamp, end_timestamp, filter, callback): + """Queries S3 based on specified "user_id", and the specified + array of ISO date strings. The results are filtered according to + the regular expression "filter". Returns an array of {filename, + URL} objects for each date in "iso_dates". + """ + obj_store = ObjectStore.GetInstance(ObjectStore.USER_LOG) + + def _OnListDates(date_listings): + """Assemble {filename, url} objects for each date listing.""" + filter_re = re.compile(filter or '.*') + callback([{'filename': key, 'url': obj_store.GenerateUrl(key)} + for logs in date_listings for key in logs if filter_re.search(key)]) + + with util.ArrayBarrier(_OnListDates) as b: + iso_dates = set() + t = start_timestamp + while t < end_timestamp: + iso_dates.add(ClientLog._IsoDate(t)) + t += constants.SECONDS_PER_DAY + iso_dates.add(ClientLog._IsoDate(end_timestamp)) + iso_dates = sorted(iso_dates) + + for iso_date in iso_dates: + ClientLog._ListAllKeys(obj_store, ClientLog._LogKeyPrefix(user_id, iso_date), b.Callback()) + + @classmethod + def _ListAllKeys(cls, obj_store, prefix, callback): + """Lists all keys for the given prefix, making multiple calls as necessary.""" + def _AppendResults(results, keys): + results += keys + if len(keys) < MAX_CLIENT_LOGS: + callback(results) + else: + obj_store.ListKeys(partial(_AppendResults, results), + prefix=prefix, marker=keys[-1], maxkeys=MAX_CLIENT_LOGS) + + obj_store.ListKeys(partial(_AppendResults, []), prefix=prefix, maxkeys=MAX_CLIENT_LOGS) diff --git a/backend/db/comment.py b/backend/db/comment.py new file mode 100644 index 0000000..da05f13 --- /dev/null +++ b/backend/db/comment.py @@ -0,0 +1,89 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder comment. + +A viewpoint may contain any number of comments, which are textual messages +contributed by followers of that viewpoint. Comments are ordered by +ascending timestamp, and uniquely qualified by device id and a device- +generated comment id. Each time a new comment is posted to a viewpoint, +a viewpoint activity is created to track the action, and a notification +is sent to each follower of the viewpoint. + +A comment can optionally be linked to other assets in the viewpoint via +its "asset_id" attribute. If a user comments on a photo, then the comment +is linked to that photo. If a user responds to a previous comment, then +the new comment is linked to the previous comment. Multiple comments can +be linked to the same "parent" comment. + + Comment: user-provided message regarding a viewpoint. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +from tornado import gen, escape +from viewfinder.backend.base.exceptions import LimitExceededError, PermissionError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.asset_id import IdPrefix, ConstructTimestampAssetId, DeconstructTimestampAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class Comment(DBRangeObject): + """Viewfinder comment data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.COMMENT) + + COMMENT_SIZE_LIMIT_BYTES = 32 * 1024 + """Max length (in bytes) of comments. + This is based on a max bytes per row in dynamo of 64KB. + """ + + def __init__(self, viewpoint_id=None, comment_id=None): + super(Comment, self).__init__() + self.viewpoint_id = viewpoint_id + self.comment_id = comment_id + + @classmethod + def ShouldScrubColumn(cls, name): + return name == 'message' + + @classmethod + def ConstructCommentId(cls, timestamp, device_id, uniquifier): + """Returns a comment id constructed from component parts. Comments + sort from oldest to newest. See "ConstructTimestampAssetId" for + details of the encoding. + """ + return ConstructTimestampAssetId(IdPrefix.Comment, timestamp, device_id, uniquifier, reverse_ts=False) + + @classmethod + def DeconstructCommentId(cls, comment_id): + """Returns the components of a comment id: timestamp, device_id, and + uniquifier. + """ + return DeconstructTimestampAssetId(IdPrefix.Comment, comment_id, reverse_ts=False) + + @classmethod + @gen.coroutine + def VerifyCommentId(cls, client, user_id, device_id, comment_id): + """Ensures that a client-provided comment id is valid according + to the rules specified in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Comment, comment_id, has_timestamp=True) + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **cm_dict): + """Creates the comment specified by "cm_dict". The caller is responsible for checking + permission to do this, as well as ensuring that the comment does not yet exist (or is + just being identically rewritten). + + Returns the created comment. + """ + comment = Comment.CreateFromKeywords(**cm_dict) + yield gen.Task(comment.Update, client) + raise gen.Return(comment) diff --git a/backend/db/contact.py b/backend/db/contact.py new file mode 100644 index 0000000..6647d6b --- /dev/null +++ b/backend/db/contact.py @@ -0,0 +1,214 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder contact. + + Contact: contact information for a user account +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import base64 +import hashlib + +from tornado import gen + +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + +@DBObject.map_table_attributes +class Contact(DBRangeObject): + """Viewfinder contact data object.""" + __slots__ = [] + + FACEBOOK = 'fb' + """Facebook contact source.""" + + GMAIL = 'gm' + """GMail contact source.""" + + IPHONE = 'ip' + """iPhone contact source.""" + + MANUAL = 'm' + """Manual contact source.""" + + ALL_SOURCES = [FACEBOOK, GMAIL, IPHONE, MANUAL] + UPLOAD_SOURCES = [IPHONE, MANUAL] + + MAX_CONTACTS_LIMIT = 10000 + """Max allowed contacts on the server per user (including removed contacts).""" + + MAX_REMOVED_CONTACTS_LIMIT = 1000 + """Number of removed contacts which will trigger garbage collection of removed contacts.""" + + REMOVED = 'removed' + """This contact has been removed from the user's address book.""" + + _table = DBObject._schema.GetTable(vf_schema.CONTACT) + + def __init__(self, user_id=None, sort_key=None): + super(Contact, self).__init__() + self.user_id = user_id + self.sort_key = sort_key + + def IsRemoved(self): + """Returns True if the contact has the Contact.REMOVED label.""" + return Contact.REMOVED in self.labels + + def Update(self, client, callback, expected=None, replace=True, return_col_names=False): + """Intercept base Update method to ensure that contact_id and sort_key are valid and correct for + current attribute values.""" + self._AssertValid() + super(Contact, self).Update(client, + callback, + expected=expected, + replace=replace, + return_col_names=return_col_names) + + @classmethod + def CalculateContactEncodedDigest(cls, **dict_to_hash): + """Calculate an encoded digest based on the dictionary passed in. The result is suitable for use + in constructing the contact_id. + """ + json_to_hash = util.ToCanonicalJSON(dict_to_hash) + m = hashlib.sha256() + m.update(json_to_hash) + base64_encoded_digest = base64.b64encode(m.digest()) + # Just use half of the base64 encoded digest to save some space. It will still be quite unique and the + # design of the contact_id only depends on uniqueness for reasonable performance, not correctness. + return base64_encoded_digest[:len(base64_encoded_digest) / 2] + + @classmethod + def CalculateContactId(cls, contact_dict): + """Calculate hash from contact dictionary.""" + # We explicitly don't hash identity(deprecated), identities(only present for indexing), + # contact_source(explicitly part of contact_id), contact_id, sort_key, and user_id + assert contact_dict.has_key('contact_source'), contact_dict + assert contact_dict.has_key('identities_properties'), contact_dict + assert contact_dict['contact_source'] in Contact.ALL_SOURCES, contact_dict + for identity_properties in contact_dict['identities_properties']: + assert len(identity_properties) <= 2, contact_dict + + dict_to_hash = {'name': contact_dict.get('name', None), + 'given_name': contact_dict.get('given_name', None), + 'family_name': contact_dict.get('family_name', None), + 'rank': contact_dict.get('rank', None), + 'identities_properties': contact_dict.get('identities_properties')} + return contact_dict.get('contact_source') + ':' + Contact.CalculateContactEncodedDigest(**dict_to_hash) + + @classmethod + def CreateContactDict(cls, user_id, identities_properties, timestamp, contact_source, **kwargs): + """Creates a dict with all properties needed for a contact. + The identities_properties parameter is a list of tuples where each tuple is: + (identity_key, description_string). Description string is for 'work', 'mobile', 'home', etc... + designation and may be None. + This includes calculation of the contact_id and sort_key from timestamp, contact_source, and other attributes. + Returns: contact dictionary. + """ + from viewfinder.backend.db.identity import Identity + + contact_dict = {'user_id': user_id, + 'timestamp': timestamp, + 'contact_source': contact_source} + + if Contact.REMOVED not in kwargs.get('labels', []): + # identities is the unique set of canonicalized identities associated with this contact. + contact_dict['identities'] = {Identity.Canonicalize(identity_properties[0]) + for identity_properties in identities_properties} + contact_dict['identities_properties'] = identities_properties + + contact_dict.update(kwargs) + if 'contact_id' not in contact_dict: + contact_dict['contact_id'] = Contact.CalculateContactId(contact_dict) + if 'sort_key' not in contact_dict: + contact_dict['sort_key'] = Contact.CreateSortKey(contact_dict['contact_id'], timestamp) + + return contact_dict + + @classmethod + def CreateFromKeywords(cls, user_id, identities_properties, timestamp, contact_source, **kwargs): + """Override base CreateWithKeywords which ensures contact_id and sort_key are defined if not provided + by the caller. + Returns: Contact object.""" + contact_dict = Contact.CreateContactDict(user_id, + identities_properties, + timestamp, + contact_source, + **kwargs) + return super(Contact, cls).CreateFromKeywords(**contact_dict) + + @classmethod + def CreateRemovedContact(cls, user_id, contact_id, timestamp): + """Create instance of a removed contact for given user_id, contact_id, and timestamp.""" + removed_contact_dict = {'user_id': user_id, + 'identities_properties': None, + 'timestamp': timestamp, + 'contact_source': Contact.GetContactSourceFromContactId(contact_id), + 'contact_id': contact_id, + 'sort_key': Contact.CreateSortKey(contact_id, timestamp), + 'labels': [Contact.REMOVED]} + return Contact.CreateFromKeywords(**removed_contact_dict) + + @classmethod + def CreateSortKey(cls, contact_id, timestamp): + """Create value for sort_key attribute. This is derived from timestamp and contact_id.""" + prefix = util.CreateSortKeyPrefix(timestamp, randomness=False) + return prefix + (contact_id if contact_id is not None else '') + + @classmethod + @gen.coroutine + def DeleteDuplicates(cls, client, contacts): + """Given list of contacts, delete any duplicates (preserving the newer contact). + Returns: list of retained contacts. + """ + contacts_dict = dict() + tasks = [] + for contact in contacts: + if contact.contact_id in contacts_dict: + if contact.timestamp > contacts_dict[contact.contact_id].timestamp: + # Delete the one in dictionary and keep the current one. + contact_to_delete = contacts_dict[contact.contact_id] + contacts_dict[contact.contact_id] = contact + else: + # Keep the one in the dictionary and delete the current one. + contact_to_delete = contact + tasks.append(gen.Task(contact_to_delete.Delete, client)) + else: + contacts_dict[contact.contact_id] = contact + + yield tasks + + raise gen.Return(contacts_dict.values()) + + @classmethod + def GetContactSourceFromContactId(cls, contact_id): + """Return the contact_id prefix which is the contact_source.""" + return contact_id.split(':', 1)[0] + + @classmethod + @gen.coroutine + def VisitContactUserIds(cls, client, contact_identity_key, visitor, consistent_read=False): + """Visits all users that have the given identity among their contacts. Invokes the + "visitor" function with each user id. See VisitIndexKeys for additional detail. + """ + def _VisitContact(contact_key, callback): + visitor(contact_key.hash_key, callback=callback) + + query_expr = ('contact.identities={id}', {'id': contact_identity_key}) + yield gen.Task(Contact.VisitIndexKeys, client, query_expr, _VisitContact) + + def _AssertValid(self): + """Assert that contact_id and sort_key are valid and correct for the current contact attributes.""" + # CalculateContactId will assert several things, too. + if self.IsRemoved(): + # A removed contact is not expected to have a contact_id calculated from it's details + # because removed contacts are stored without contact details. + assert self.contact_source is not None and self.contact_source in Contact.ALL_SOURCES, self + else: + contact_id = Contact.CalculateContactId(self._asdict()) + assert contact_id == self.contact_id, self + assert self.timestamp is not None, self + sort_key = Contact.CreateSortKey(self.contact_id, self.timestamp) + assert sort_key == self.sort_key, self diff --git a/backend/db/db_client.py b/backend/db/db_client.py new file mode 100644 index 0000000..93cfc86 --- /dev/null +++ b/backend/db/db_client.py @@ -0,0 +1,253 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Interface for client access to datastore backends. + +Implemented via the DynamoDB client (dynamodb_client) and the local datastore +emulation client (local_client). + +Each client operation takes a callback for asynchronous operation. + + Client operations: + - GetItem: retrieve a database item by key (can be composite key) + - BatchGetItem: retrieve a batch of database items by key + - PutItem: store a database item + - DeleteItem: deletes a database item + - UpdateItem: update attributes of a database item + - Query: queries database item(s) +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from collections import namedtuple +from tornado import ioloop, options + +options.define('localdb', default=False, help='use local datastore emulation') +options.define('localdb_dir', default='./local/db', + help='directory in which to store database persistence files') +options.define('localdb_sync_secs', default=1.0, + help='seconds between successive syncs to disk') +options.define('localdb_version', default=0, + help='specify a version other than 0 to use as the current on startup; ' + 'the current version ".0" is still moved to ".1" as normal') +options.define('localdb_num_versions', default=20, + help='number of previous versions of the database to maintain') +options.define('localdb_reset', default=False, + help='reset all existing database files') + +options.define('readonly_db', default=False, help='Read-only database') + + +# Operation information, including operation id and priority, 'op_id' +# == 0 means the request is not attached to an operation but is being +# made extemporaneously. +DBOp = namedtuple('DBOp', ['op_id', 'priority']) + +# Named tuple for database keys. Composite keys define both the hash +# key and the range key. Objects which have only a hash key leave the +# range key as None. +DBKey = namedtuple('DBKey', ['hash_key', 'range_key']) + +DBKeySchema = namedtuple('DBKeySchema', ['name', 'value_type']) + +# Named tuple of calls to Client.UpdateItem. Action must be one of +# 'PUT', 'ADD', 'DELETE'. +UpdateAttr = namedtuple('UpdateAttr', ['value', 'action']) + +# Named tuple for range key queries. 'key' is a list of length 1 if +# 'op' is one of (EQ|LE|LT|GE|GT|BEGINS_WITH), --or-- 'key' is a list +# of length 2 ([start, end]), if 'op' is BETWEEN. +RangeOperator = namedtuple('RangeOperator', ['key', 'op']) + +# Named tuple for scan filter. The comments for RangeOperator apply +# here as well, though local_db supports only a subset of the actual +# scan filter functionality. 'value' is analagous here to 'key' in +# RangeOperator. It is a list of either one or more than one values +# depending on the value of 'op'. +ScanFilter = namedtuple('ScanFilter', ['value', 'op']) + +# Description of a table. +TableSchema = namedtuple('TableSchema', ['create_time', 'hash_key_schema', + 'range_key_schema', 'read_units', + 'write_units', 'status']) + +# Table metadata results. +ListTablesResult = namedtuple('ListTables', ['tables']) +CreateTableResult = namedtuple('CreateTable', ['schema']) +DescribeTableResult = namedtuple('DescribeTable', ['schema', 'count', 'size_bytes']) +DeleteTableResult = namedtuple('DeleteTable', ['schema']) + +# Named tuples for results of datastore operations. +GetResult = namedtuple('GetResult', ['attributes', 'read_units']) +PutResult = namedtuple('PutResult', ['return_values', 'write_units']) +DeleteResult = namedtuple('DeleteResult', ['return_values', 'write_units']) +UpdateResult = namedtuple('UpdateResult', ['return_values', 'write_units']) +QueryResult = namedtuple('QueryResult', ['count', 'items', 'last_key', 'read_units']) +ScanResult = namedtuple('ScanResult', ['count', 'items', 'last_key', 'read_units']) + +# Batch tuples (batch operations use dictionary that maps from table name => tuple). +BatchGetRequest = namedtuple('BatchGetRequest', ['keys', 'attributes', 'consistent_read']) +BatchGetResult = namedtuple('BatchGetResult', ['items', 'read_units']) + + +class DBClient(object): + """Interface for asynchronous access to backend datastore. + """ + def Shutdown(self): + """Cleanup on process exit.""" + raise NotImplementedError() + + def ListTables(self, callback): + """Lists the set of tables.""" + raise NotImplementedError() + + def CreateTable(self, table, hash_key_schema, range_key_schema, + read_units, write_units, callback): + """Create a table with specified name, key schema and provisioned + throughput settings. + """ + raise NotImplementedError() + + def DeleteTable(self, table, callback): + """Create a table with specified name, key schema and provisioned + throughput settings. + """ + raise NotImplementedError() + + def DescribeTable(self, table, callback): + """Describes the named table.""" + raise NotImplementedError() + + def GetItem(self, table, key, callback, attributes, must_exist=True, + consistent_read=False): + """Gets the specified attribute values by key. 'must_exist' + specifies whether to throw an exception if the item is not found. + If False, None is returned if not found. 'consistent_read' + designates whether to fetch an authoritative value for the item. + """ + raise NotImplementedError() + + def BatchGetItem(self, batch_dict, callback, must_exist=True): + """Gets a batch of items from the database. Items to get are described in 'batch_dict', + which has the following format: + + {'table-name-0': BatchGetRequest(keys=, + attributes=[attr-0, attr-1, ...], + consistent_read=), + 'table-name-1': ...} + + Returns results in the following format: + + {'table-name-0': BatchGetResult(items={'attr-0': value-0, 'attr-1': value-1, ...}, + read_units=3.0), + 'table-name-1': ...} + + If 'must_exist' is true, then raises an error if a db-key is not found in the table. + Otherwise, returns None in corresponding positions in the 'items' array. + """ + raise NotImplementedError() + + def PutItem(self, table, key, callback, attributes, expected=None, + return_values=None): + """Sets the specified item attributes by key. 'attributes' is a + dict {attr: value}. If 'expected' is not None, requires that the + values specified in the expected dict {attr: value} match before + mutation. 'return_values', if not None, must be one of (NONE, + ALL_OLD); if ALL_OLD, the previous values for the named attributes + are returned as an attribute dict. + """ + raise NotImplementedError() + + def DeleteItem(self, table, key, callback, expected=None, + return_values=None): + """Deletes the specified item by key. 'expected' and + 'return_values' are identical to PutItem(). + """ + raise NotImplementedError() + + def UpdateItem(self, table, key, callback, attributes, expected=None, + return_values=None): + """Updates the specified item attributes by key. 'attributes' is a + dict {attr: AttrUpdate} (see AttrUpdate named tuple above). + 'expected' and 'return_values' are the same as for PutItem(), + except that 'return_values' may contain any of (NONE, ALL_OLD, + UPDATED_OLD, ALL_NEW, UPDATED_NEW). + """ + raise NotImplementedError() + + def Query(self, table, hash_key, range_operator, callback, attributes, + limit=None, consistent_read=False, count=False, + scan_forward=True, excl_start_key=None): + """Queries a range of values by 'hash_key' and 'range_operator'. + 'range_operator' is of type RangeOperator (see named tuple above; + if None, selects all values). 'attributes' is a list of + attributes to query, limit is an upper limit on the number of + results. If True, 'count' will return just a count of items, but + no actual data. 'scan_forward', if False, causes a reverse scan + according to the range operator. If not None, 'excl_start_key' + allows the query operation to start partway through the + range. 'excl_start_key' specifies just the range key. + """ + raise NotImplementedError() + + def Scan(self, table, callback, attributes, limit=None, + excl_start_key=None, scan_filter=None): + """Scans the table starting at 'excl_start_key' (if provided) and + reading the next 'limit' rows, reading the specified 'attributes'. + If 'scan_filter' is specified, it is applied to each scanned item + to pre-filter returned results. 'scan_filter' is a map from + attribute name to ScanFilter tuple. + """ + raise NotImplementedError() + + def AddTimeout(self, deadline_secs, callback): + """Invokes the specified callback after 'deadline_secs'. Returns a + handle which can be suppled to RemoveTimeout to disable the + timeout. + """ + raise NotImplementedError() + + def AddAbsoluteTimeout(self, abs_timeout, callback): + """Invokes the specified callback at wall time + 'abs_timeout'. Returns a handle which can be supplied to + RemoveTimeout to disable the timeout.""" + raise NotImplementedError() + + def RemoveTimeout(self, timeout): + """Removes a timeout added via AddTimeout or AddAbsoluteTimeout.""" + raise NotImplementedError() + + @staticmethod + def Instance(): + assert hasattr(DBClient, "_instance"), 'instance not initialized' + return DBClient._instance + + @staticmethod + def SetInstance(client): + """Sets a new instance for testing.""" + DBClient._instance = client + + +def InitDB(schema=None, callback=None, verify_or_create=True): + """Sets the db client instance. + Initialize the local datastore if --localdb was specified. + + Callback is invoked with the verified table schemas if + 'verify_or_create' is True; None otherwise. + """ + assert not hasattr(DBClient, "_instance"), 'instance already initialized' + assert schema is not None + if options.options.localdb: + from local_client import LocalClient + DBClient.SetInstance(LocalClient(schema, read_only=options.options.readonly_db)) + else: + from dynamodb_client import DynamoDBClient + DBClient._instance = DynamoDBClient(schema, read_only=options.options.readonly_db) + if verify_or_create: + schema.VerifyOrCreate(DBClient.Instance(), callback) + else: + callback([]) + +def ShutdownDB(): + """Shuts down the currently running instance.""" + if hasattr(DBClient, "_instance"): + DBClient.Instance().Shutdown() diff --git a/backend/db/db_import.py b/backend/db/db_import.py new file mode 100644 index 0000000..3c4d390 --- /dev/null +++ b/backend/db/db_import.py @@ -0,0 +1,27 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Imports all database tables and sets up a mapping from table +name to table object class. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import re +from tornado.util import import_object + +# List tables here if they do not follow our naming convention. +TABLE_ALIAS_MAP = { + 'Index': None, + 'TestRename': 'viewfinder.backend.db.test.test_rename.TestRename', +} + +def GetTableClass(class_name): + if class_name in TABLE_ALIAS_MAP: + qualified_name = TABLE_ALIAS_MAP[class_name] + else: + # Convert CamelCase to underscore_separated. + package_name = re.sub(r'(.)([A-Z])', r'\1_\2', class_name).lower() + qualified_name = 'viewfinder.backend.db.%s.%s' % (package_name, class_name) + if qualified_name is None: + return None + return import_object(qualified_name) diff --git a/backend/db/device.py b/backend/db/device.py new file mode 100644 index 0000000..33eb015 --- /dev/null +++ b/backend/db/device.py @@ -0,0 +1,266 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Device description. + +Describes a mobile device. Each device is tightly coupled to the user +id that first registers it. On registration, a device is assigned a +unique id from the id allocation table, which when combined with ids +assigned either locally by the device, or through each user's id_seq +column (in the case of access via the web application), provides a +stream of unique photo and episode ids. + +A device is created and associated with a user on registration. If +registration is done on the web application, no device is stored with +the user cookie. In this case, photo ids are allocated through the +user's 'id_seq' sequence. With a mobile device, platform, os & +Viewfinder app version are all stored with the device. The device +itself manages the photo id sequence. + +On a new registration, no device id is provided by the mobile app. It +is generated from the id allocation table and returned with a +successful authentication / authorization. However, the device does +provide information on app version, os & platform. On subsequent +registrations, the device id is provided along with any updates to +device information. The device id is stored in the secure user cookie +and is supplied on every subsequent request. + +Each device may have an associated 'push_token' (ex. APNs token for +iOS devices). Every time the device is used and a registration request +is sent to viewfinder, the push_token--if applicable--is supplied and +the 'last_access' timestamp is updated. 'alert' and 'badge' are set +when notifications are generated in response to activity on a user +account. 'alert' is a message to display with a push notification; +'badge' is a number indicating the number of pending updates to the +user's account. + + Device: device information; mobile (iOS, Android, etc.) or web-app +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import time + +from copy import deepcopy +from functools import partial +from tornado import gen +from viewfinder.backend.base import constants, util +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.id_allocator import IdAllocator +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.db.settings import AccountSettings +from viewfinder.backend.services.push_notification import PushNotification + + +@DBObject.map_table_attributes +class Device(DBRangeObject): + """Viewfinder device data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.DEVICE) + + _ALLOCATION = 1 + _allocator = IdAllocator(id_type=_table.range_key_col.name, allocation=_ALLOCATION) + + _sys_obj_allocator = IdAllocator(id_type='system-object-id', allocation=_ALLOCATION) + """Used to allocate ids for objects created by the system (e.g. + viewpoint created during migration of user episodes). + """ + + _MAX_ALERT_DEVICES = 25 + """Maximum number of devices to which Viewfinder will push alerts.""" + + SYSTEM = 0 + """Id of reserved system device, used when allocating system objects.""" + + def __init__(self, user_id=None, device_id=None): + super(Device, self).__init__() + self.user_id = user_id + self.device_id = device_id + + @classmethod + def Create(cls, **device_dict): + """Create a new Device object from 'device_dict'. Clears out 'device_uuid' and 'test_udid' if present.""" + create_dict = device_dict + if 'device_uuid' in device_dict or 'test_udid' in device_dict: + create_dict = deepcopy(device_dict) + create_dict.pop('device_uuid', None) + create_dict.pop('test_udid', None) + return cls.CreateFromKeywords(**create_dict) + + def UpdateFields(self, **device_dict): + """Update a Device object from 'device_dict'. Clears out 'device_uuid' and 'test_udid' if present.""" + create_dict = device_dict + if 'device_uuid' in device_dict or 'test_udid' in device_dict: + create_dict = deepcopy(device_dict) + create_dict.pop('device_uuid', None) + create_dict.pop('test_udid', None) + self.UpdateFromKeywords(**create_dict) + + @classmethod + def ShouldScrubColumn(cls, name): + return name == 'name' + + @classmethod + @gen.coroutine + def Register(cls, client, user_id, device_dict, is_first=True): + """Registers a new device or update an existing device, using the fields in "device_dict". + If "is_first" is true, then this is the first mobile device to be registered for this + user. + """ + assert 'device_id' in device_dict, device_dict + + device = yield gen.Task(Device.Query, + client, + user_id, + device_dict['device_id'], + None, + must_exist=False) + if device is None: + device = Device.Create(user_id=user_id, timestamp=util.GetCurrentTimestamp(), **device_dict) + else: + device.UpdateFields(**device_dict) + + yield gen.Task(device.Update, client) + + # If this is the first mobile device to be registered, then turn turn off email alerting + # and turn on full push alerting to mobile devices. + if is_first: + settings = AccountSettings.CreateForUser(user_id, + email_alerts=AccountSettings.EMAIL_NONE, + sms_alerts=AccountSettings.SMS_NONE, + push_alerts=AccountSettings.PUSH_ALL) + yield gen.Task(settings.Update, client) + + raise gen.Return(device) + + def Update(self, client, callback): + """Call the base class "Update" method in order to persist modified + columns to the db. But also ensure that this device has a unique + push token; two Viewfinder devices might share the same push token + if a phone has been given or sold to another person without + re-installing the OS. Also, ensure that the device is added to the + secondary index used for alerting (alert_user_id), for fast + enumeration of all devices that need to be alerted for a particular + user. + """ + def _DoUpdate(): + super(Device, self).Update(client, callback) + + def _OnQueryByPushToken(devices): + """Disable alerts for all other devices.""" + with util.Barrier(_DoUpdate) as b: + for device in devices: + if device.device_id != self.device_id: + device.push_token = None + device.alert_user_id = None + super(Device, device).Update(client, b.Callback()) + + # Each time the device is updated, update the last_access field. + self.last_access = util.GetCurrentTimestamp() + + if self._IsModified('push_token'): + if self.push_token is None: + self.alert_user_id = None + _DoUpdate() + else: + # Ensure that the device will be alerted. + self.alert_user_id = self.user_id + + query_expr = ('device.push_token={t}', {'t': self.push_token}) + Device.IndexQuery(client, query_expr, None, _OnQueryByPushToken) + else: + _DoUpdate() + + @classmethod + def PushNotification(cls, client, user_id, alert, badge, callback, + exclude_device_id=None, extra=None, sound=None): + """Queries all devices for 'user'. Devices with 'push_token' + set are pushed notifications via the push_notification API. + NOTE: currently, code path is synchronous, but the callback + is provided in case that changes. + + If specified, 'exclude_device_id' will exclude a particular device + from the set to which notifications are pushed. For example, the + device which is querying notifications when the badge is set to 0. + """ + def _OnQuery(devices): + with util.Barrier(callback) as b: + now = util.GetCurrentTimestamp() + for device in devices: + if device.device_id != exclude_device_id: + token = device.push_token + assert token, device + try: + PushNotification.Push(token, alert=alert, badge=badge, sound=sound, extra=extra) + except TypeError as e: + logging.error('bad push token %s', token) + Device._HandleBadPushToken(client, token, time.time(), b.Callback()) + except Exception as e: + logging.warning('failed to push notification to user %d: %s', user_id, e) + raise + + # Find all devices owned by the user that need to be alerted. + Device.QueryAlertable(client, user_id, _OnQuery) + + @classmethod + def QueryAlertable(cls, client, user_id, callback, limit=_MAX_ALERT_DEVICES): + """Returns all devices owned by the given user that can be alerted.""" + query_expr = ('device.alert_user_id={u}', {'u': user_id}) + Device.IndexQuery(client, query_expr, None, callback, limit=limit) + + @classmethod + @gen.coroutine + def MuteAlerts(cls, client, user_id): + """Turn off alerts to all devices owned by "user_id".""" + @gen.coroutine + def _VisitDevice(device): + device.alert_user_id = None + yield gen.Task(device.Update, client) + + yield gen.Task(Device.VisitRange, client, user_id, None, None, _VisitDevice) + + @classmethod + def FeedbackHandler(cls, client): + """Returns a callback which deals appropriately with device push + tokens which have failed delivery. + """ + return partial(Device._HandleBadPushToken, client) + + @classmethod + def AllocateSystemObjectId(cls, client, callback): + """Generate a unique id to be used for identifying system-generated + objects. Return the new id. + """ + Device._sys_obj_allocator.NextId(client, callback) + + @classmethod + def _HandleBadPushToken(cls, client, push_token, timestamp=None, callback=None): + """Callback in the event of failed delivery of push notification. + 'push_token' is queried via the secondary index on Device. Timestamp + is the time at which the delivery failed. If the device attached to + the failed push_token has 'last_access' > timestamp, ignore failure; + otherwise, clear the push token and update. + """ + def _OnQueryByPushToken(devices): + if not devices: + logging.warning('unable to locate device for push token: %s' % push_token) + return + for device in devices: + if device.last_access is None or device.last_access < timestamp: + logging.info('unsetting push_token for device %s' % device) + device.push_token = None + device.Update(client, callback if callback else util.NoCallback) + + query_expr = ('device.push_token={t}', {'t': push_token}) + Device.IndexQuery(client, query_expr, None, _OnQueryByPushToken) + + @classmethod + def UpdateOperation(cls, client, callback, user_id, device_id, device_dict): + """Updates device metadata.""" + def _OnQuery(device): + device.UpdateFields(**device_dict) + device.Update(client, callback) + + Device.Query(client, user_id, device_id, None, _OnQuery) diff --git a/backend/db/dynamodb_client.py b/backend/db/dynamodb_client.py new file mode 100644 index 0000000..a1699a6 --- /dev/null +++ b/backend/db/dynamodb_client.py @@ -0,0 +1,700 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Client access to DynamoDB backend. + +The client marshals and unmarshals Viewfinder schema objects and +parameters to/from the DynamoDB JSON-encoded format. + + DynamoDBClient +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import heapq +import json +import logging +import time + +from boto.exception import DynamoDBResponseError +from collections import namedtuple +from functools import partial +from tornado import gen, ioloop, stack_context +from viewfinder.backend.base import secrets, util, counters, rate_limiter +from viewfinder.backend.base.exceptions import DBProvisioningExceededError, DBLimitExceededError +from viewfinder.backend.base.util import ConvertToString, ConvertToNumber +from viewfinder.backend.db.asyncdynamo import AsyncDynamoDB +from db_client import DBClient, DBKey, ListTablesResult, CreateTableResult, DescribeTableResult, DeleteTableResult, GetResult, PutResult, DeleteResult, UpdateResult, QueryResult, ScanResult, BatchGetResult, DBKeySchema, TableSchema + +# List of tables for which we want to save qps/backoff metrics. +kSaveMetricsFor = ['Follower', 'Photo', 'Viewpoint'] + +# The maximum throttling allowed as a fraction of the throughput capacity. 0.75 means that we can be limited down +# by at most 75% of the capacity (eg: we'll be allowed to use 1/4). +# TODO(marc): this should really take the number of backends into account. +kMaxCapacityThrottleFraction = 0.75 + +# What fraction of the throughput capacity do we lose every time we receive a throttle from dynamodb. +# eg: 0.05 means that we'll lose 5% of our capacity every time. +kPerThrottleLostCapacityFraction = 0.1 + +# Minimum amount of time between rate adjustments, in seconds. +kMinRateAdjustmentPeriod = 1.0 + + +DynDBRequest = namedtuple('DynDBRequest', ['method', 'request', 'op', 'execute_cb', 'finish_cb']) + +_requests_queued = counters.define_total('viewfinder.dynamodb.requests_queued', + 'Number of DynamoDB requests currently queued.') +# TODO: we should have this per table. A global counter is mostly meaningless. +_throttles_per_min = counters.define_rate('viewfinder.dynamodb.throttles_per_min', + 'Number of throttling errors received from DynamoDB per minute.', 60) +# In addition to these counters, each RequestQueue may setup an extra two (one for QPS, one for backoff). + + +class RequestQueue(object): + """Manages the complexity of tracking successes and failures and + estimating backoff delays for a request queue. + """ + + def __init__(self, table_name, read_write, name, ups): + """'ups' is measured either as read or write capacity units per second. + """ + self._name = name + self._ups = ups + self._queue = [] + self._last_rate_adjust = time.time() + self._unavailable_rate = 0.0 + self._need_adj = False + + qps_counter = backoff_counter = None + if table_name in kSaveMetricsFor: + rw_str = 'write' if read_write else 'read' + qps_counter = counters.define_rate('viewfinder.dynamodb.qps.%s_%s' % (table_name, rw_str), + 'Dynamodb %s QPS on %s' % (rw_str, table_name), 1) + backoff_counter = counters.define_rate('viewfinder.dynamodb.backoff_per_sec.%s_%s' % (table_name, rw_str), + 'Dynamodb %s backoff seconds per second on %s' % (rw_str, table_name), 1) + self._ups_rate = rate_limiter.RateLimiter(ups, qps_counter=qps_counter, backoff_counter=backoff_counter) + + self._timeout = None + + def Push(self, req): + """Adds 'req', a DynDBRequest tuple, to the priority queue. + """ + _requests_queued.increment() + heapq.heappush(self._queue, (self._ComputePriority(req), req)) + + def Pop(self): + """Pops the highest priority request from the queue and returns it.""" + self._ups_rate.Add(1.0) + _requests_queued.decrement() + return heapq.heappop(self._queue)[1] + + def IsEmpty(self): + """Returns True if the queue is empty, False otherwise.""" + return len(self._queue) == 0 + + def Report(self, success, units=1): + """'success' specifies whether or not the request failed due to a + provisioned throughput exceeded error. On success, we adjust + self._ups_stat if units != 1, as in the case of an eventually + consistent read (units=0.5), or an operation requiring more than + 1 unit. + + On failure, set the _need_adj flag. + """ + if success and units != 1.0: + logging.debug('reported %.2f units for queue %s' % (units, self._name)) + self._ups_rate.Add(units - 1.0) + + if not success: + _throttles_per_min.increment() + self._need_adj = True + + def RecomputeRate(self): + """Adjust the unavailable qps if needed and it's been long enough since the last adjustment. + Increase or decrease based on the _need_adj flag and current min/max. + """ + now = time.time() + if (now - self._last_rate_adjust) >= kMinRateAdjustmentPeriod: + new_adj = None + # It's been long enough, we can adjust the rate if needed. + if self._need_adj and self._unavailable_rate < (self._ups * kMaxCapacityThrottleFraction): + new_adj = self._ups * kPerThrottleLostCapacityFraction + elif not self._need_adj and self._unavailable_rate > 0.0: + new_adj = -self._ups * kPerThrottleLostCapacityFraction + # Clear need_adj regardless of whether we can change the rate or not. Otherwise, it will never be cleared + # when we hit the max value for 'unavailable qps'. + self._need_adj = False + + if new_adj is not None: + self._last_rate_adjust = now + self._unavailable_rate += new_adj + self._ups_rate.SetUnavailableQPS(self._unavailable_rate) + + def GetBackoffSecs(self): + """Ask the rate limiter for the number of seconds to sleep. We must sleep this long. + We do not call RecomputeRate here since NeedsBackoff just did it. + """ + return self._ups_rate.ComputeBackoffSecs() + + def NeedsBackoff(self): + """Returns whether or not this queue needs to backoff. This calls a method on the rate limiter that does not + increment the backoff counter. + We first recompute the unavailable rate and adjust it if needed. + """ + self.RecomputeRate() + return self._ups_rate.NeedsBackoff() + + def ResetTimeout(self, callback): + """Clears any existing timeout registered on the ioloop for this + queue. If there is a current backoff and the queue is not empty, + sets a new timeout based on backoff. + """ + def _OnTimeout(): + self._timeout = None + callback() + + if not self.IsEmpty() and self._timeout is None: + backoff_secs = self.GetBackoffSecs() + self._timeout = ioloop.IOLoop.current().add_timeout(time.time() + backoff_secs, _OnTimeout) + elif self.IsEmpty(): + if self._timeout: + ioloop.IOLoop.current().remove_timeout(self._timeout) + self._timeout = None + + def _ComputePriority(self, req): + """Computes the priority of 'req'. First cut of this algorithm is + to simply order by the time the request was (re)added to the queue. + """ + return time.time() + + +class RequestScheduler(object): + """Prioritizes and schedules competing requests to the DynamoDB + backend. Requests are organized by tables. Each table has its own + provisioning for read and writes per second. Depending on failures + indicating that provisioned throughput is being exceeded, requests + are placed into priority queues and throttled to just under the + maximum sustainable rate. + """ + _READ_ONLY_METHODS = ('ListTables', 'DescribeTable', 'GetItem', 'Query', 'Scan', 'BatchGetItem') + + def __init__(self, schema): + self._read_queues = dict([(t.name_in_db, RequestQueue(t.name, False, '%s reads' % (t.name), t.read_units)) \ + for t in schema.GetTables()]) + self._write_queues = dict([(t.name_in_db, RequestQueue(t.name, True, '%s writes' % (t.name), t.write_units)) \ + for t in schema.GetTables()]) + self._cp_read_only_queue = RequestQueue('ControlPlane', False, 'Control Plane R/O', 100) + self._cp_mutate_queue = RequestQueue('ControlPlane', True, 'Control Plane Mutate', 1) + self._paused = False + self._asyncdynamo = AsyncDynamoDB(secrets.GetSecret('aws_access_key_id'), + secrets.GetSecret('aws_secret_access_key')) + + def Schedule(self, method, request, callback): + """Creates a DynamoDB request to API call 'method' with JSON + encoded arguments 'request'. Invokes 'callback' with JSON decoded + response as an argument. + """ + if method in ('ListTables', 'DescribeTable'): + queue = self._cp_read_only_queue + elif method in ('CreateTable', 'DeleteTable'): + queue = self._cp_mutate_queue + elif method in ('GetItem', 'Query', 'Scan'): + queue = self._read_queues[request['TableName']] + elif method in ('BatchGetItem',): + table_names = request['RequestItems'].keys() + assert len(table_names) == 1, table_names + queue = self._read_queues[table_names[0]] + else: + assert method in ('DeleteItem', 'PutItem', 'UpdateItem'), method + queue = self._write_queues[request['TableName']] + + # The execution callback that we initialize the dynamodb request with is wrapped + # so that on execution, errors will be handled in the context of this method's caller. + dyn_req = DynDBRequest(method=method, request=request, op=None, finish_cb=callback, + execute_cb=stack_context.wrap(partial(self._ExecuteRequest, queue))) + queue.Push(dyn_req) + self._ProcessQueue(queue) + + def _ExecuteRequest(self, queue, dyn_req): + """Helper function to execute a DynamoDB request within the context + in which is was scheduled. This way, if an unrecoverable exception is + thrown during execution, it can be re-raised to the appropriate caller. + """ + def _OnResponse(start_time, json_response): + if dyn_req.method in ('BatchGetItem',): + consumed_units = next(json_response.get('Responses').itervalues()).get('ConsumedCapacityUnits', 1) + else: + consumed_units = json_response.get('ConsumedCapacityUnits', 1) + + logging.debug('%s response: %d bytes, %d units, %.3fs elapsed' % + (dyn_req.method, len(json_response), consumed_units, + time.time() - start_time)) + queue.Report(True, consumed_units) + dyn_req.finish_cb(json_response) + + def _OnException(type, value, tb): + if type in (DBProvisioningExceededError, DBLimitExceededError): + # Retry on DynamoDB throttling errors. Report the failure to the queue so that it will backoff the + # requests/sec rate. + queue.Report(False) + elif type == DynamoDBResponseError and value.status in [500, 599] and \ + dyn_req.method in RequestScheduler._READ_ONLY_METHODS: + # DynamoDB returns 500 when the service is unavailable for some reason. + # Curl returns 599 when something goes wrong with the connection, such as a timeout or connection reset. + # Only retry if this is a read-only request, since otherwise an update may be applied twice. + pass + else: + # Re-raise the exception now that we're in the stack context of original caller. + logging.warning('error calling "%s" with this request: %s' % (dyn_req.method, dyn_req.request)) + raise type, value, tb + + if dyn_req.method in ('BatchGetItem',): + table_name = next(dyn_req.request['RequestItems'].iterkeys()) + else: + table_name = dyn_req.request.get('TableName', None) + logging.warning('%s against %s table failed: %s' % (dyn_req.method, table_name, value)) + queue.Push(dyn_req) + self._ProcessQueue(queue) + + logging.debug('sending %s (%d bytes) dynamodb request' % (dyn_req.method, len(dyn_req.request))) + with util.MonoBarrier(partial(_OnResponse, time.time()), on_exception=partial(_OnException)) as b: + self._asyncdynamo.make_request(dyn_req.method, json.dumps(dyn_req.request), b.Callback()) + + def _ProcessQueue(self, queue): + """If the queue is not empty and adequate provisioning is expected, + sends the highest priority queue item(s) to DynamoDB. + + When all items have been sent, resets the queue processing timeout. + """ + if self._paused: + return + + while not queue.IsEmpty() and not queue.NeedsBackoff(): + dyn_req = queue.Pop() + dyn_req.execute_cb(dyn_req) + + queue.ResetTimeout(partial(self._ProcessQueue, queue)) + + def _Pause(self): + """Pauses all queue processing. No requests will be sent until + _Resume() is invoked. + NOTE: intended for testing. + """ + self._paused = True + + def _Resume(self): + """Resume the scheduler if paused.""" + if self._paused: + self._paused = False + [self._ProcessQueue(q) for q in self._read_queues.values()] + [self._ProcessQueue(q) for q in self._write_queues.values()] + [self._ProcessQueue(q) for q in (self._cp_read_only_queue, self._cp_mutate_queue)] + + +class DynamoDBClient(DBClient): + """Asynchronous access to DynamoDB datastore. + """ + _MAX_BATCH_SIZE = 100 + """Maximum number of key rows that can be specified in a DynamoDB batch.""" + + def __init__(self, schema, read_only=False): + """Uses single ConnectionManager instance of connection_manager is None. + """ + self._schema = schema + self._read_only = read_only + self._scheduler = RequestScheduler(schema) + + def Shutdown(self): + pass + + def ListTables(self, callback): + def _OnList(response): + # Map to application table names, which may be different than names in database. + table_names = [self._schema.TranslateNameInDb(name_in_db) + for name_in_db in response['TableNames']] + callback(ListTablesResult(tables=table_names)) + + self._scheduler.Schedule('ListTables', {}, _OnList) + + def CreateTable(self, table, hash_key_schema, range_key_schema, + read_units, write_units, callback): + assert not self._read_only, 'Received "CreateTable" request on read-only database' + + def _OnCreate(response): + callback(CreateTableResult(self._GetTableSchema(table, response['TableDescription']))) + + request = { + 'TableName': table, + 'KeySchema': { + 'HashKeyElement': {'AttributeName': hash_key_schema.name, + 'AttributeType': hash_key_schema.value_type}, + }, + 'ProvisionedThroughput': {'ReadCapacityUnits': read_units, + 'WriteCapacityUnits': write_units}, + } + if range_key_schema: + request['KeySchema']['RangeKeyElement'] = { + 'AttributeName': range_key_schema.name, + 'AttributeType': range_key_schema.value_type, + } + self._scheduler.Schedule('CreateTable', request, _OnCreate) + + def DeleteTable(self, table, callback): + assert not self._read_only, 'Received "DeleteTable" request on read-only database' + + table_def = self._schema.GetTable(table) + + def _OnDelete(response): + callback(DeleteTableResult(self._GetTableSchema(table_def.name_in_db, response['TableDescription']))) + + self._scheduler.Schedule('DeleteTable', {'TableName': table_def.name_in_db}, _OnDelete) + + def DescribeTable(self, table, callback): + table_def = self._schema.GetTable(table) + + def _OnDescribe(response): + desc = response['Table'] + schema = self._GetTableSchema(table_def.name_in_db, desc) + callback(DescribeTableResult(schema=schema, + count=desc.get('ItemCount', 0), + size_bytes=desc.get('TableSizeBytes', 0))) + + self._scheduler.Schedule('DescribeTable', {'TableName': table_def.name_in_db}, _OnDescribe) + + def GetItem(self, table, key, callback, attributes, must_exist=True, + consistent_read=False): + table_def = self._schema.GetTable(table) + + def _OnGetItem(response): + if must_exist: + assert 'Item' in response, 'key %r does not exist in %s' % (key, table_def.name) + if 'Item' not in response: + callback(None) + else: + callback(GetResult(attributes=self._FromDynamoAttributes(table_def, response['Item']), + read_units=response['ConsumedCapacityUnits'])) + + request = self._GetBaseRequest(table_def, key) + request.update({'AttributesToGet': attributes, + 'ConsistentRead': consistent_read}) + self._scheduler.Schedule('GetItem', request, _OnGetItem) + + @gen.engine + def BatchGetItem(self, batch_dict, callback, must_exist=True): + """See the header for DBClient.BatchGetItem for details. Note that currently items can + only be requested from a single table at a time (though the interface supports multiple + tables). + """ + assert len(batch_dict) == 1, 'BatchGetItem currently supports only a single table' + + # Create dict of all unique keys to get. + table_name, (keys, attributes, consistent_read) = next(batch_dict.iteritems()) + table_def = self._schema.GetTable(table_name) + key_result_dict = {key: None for key in keys} + read_units = 0.0 + + # Loop until all keys have been fetched, at most 100 at a time. + while True: + item_count = 0 + dyn_keys = [] + for key, result in key_result_dict.iteritems(): + if result is None: + # By default, assume that key does not exist (will just not be returned by DynamoDB). + key_result_dict[key] = {} + dyn_keys.append(self._ToDynamoKey(table_def, key)) + item_count += 1 + + if item_count >= DynamoDBClient._MAX_BATCH_SIZE: + break + + # If no items to fetch, then done. + if item_count == 0: + break + + # Create the request dict. + request = {'RequestItems': {table_def.name_in_db: {'Keys': dyn_keys, + 'AttributesToGet': attributes, + 'ConsistentRead': consistent_read}}} + + # Send request to DynamoDB. + response = yield gen.Task(self._scheduler.Schedule, 'BatchGetItem', request) + + # Re-send any unprocessed keys by setting their results back to None. + if response['UnprocessedKeys']: + for dyn_key in response['UnprocessedKeys'][table_def.name_in_db]['Keys']: + key = self._FromDynamoKey(table_def, dyn_key) + key_result_dict[key] = None + + # Save any response attributes from items that were found by key. + for dyn_attrs in response['Responses'][table_def.name_in_db]['Items']: + dyn_key = {'HashKeyElement': dyn_attrs[table_def.hash_key_col.key]} + if table_def.range_key_col is not None: + dyn_key['RangeKeyElement'] = dyn_attrs[table_def.range_key_col.key] + + key = self._FromDynamoKey(table_def, dyn_key) + key_result_dict[key] = self._FromDynamoAttributes(table_def, dyn_attrs) + + read_units += response['Responses'][table_def.name_in_db]['ConsumedCapacityUnits'] + + # Return one item in result for each key in batch_dict. + result_items = [] + for key in next(batch_dict.itervalues()).keys: + attributes = key_result_dict[key] or None + if must_exist: + assert attributes is not None, 'key %r does not exist in %s' % (key, table_def.name) + result_items.append(attributes) + + callback({table_name: BatchGetResult(items=result_items, read_units=read_units)}) + + def PutItem(self, table, key, callback, attributes, expected=None, + return_values=None): + assert not self._read_only, 'Received "PutItem" request on read-only database' + + table_def = self._schema.GetTable(table) + + def _OnPutItem(response): + callback(PutResult(return_values=self._FromDynamoAttributes(table_def, response.get('Attributes', None)), + write_units=response['ConsumedCapacityUnits'])) + + # Add key values to the attributes map, in accordance with DynamoDB requirements. + attributes[table_def.hash_key_col.key] = key.hash_key + if table_def.range_key_col: + attributes[table_def.range_key_col.key] = key.range_key + request = {'TableName': table_def.name_in_db, + 'Item': self._ToDynamoAttributes(table_def, attributes)} + if expected is not None: + request['Expected'] = self._ToDynamoExpected(table_def, expected) + if return_values is not None: + request['ReturnValues'] = return_values + self._scheduler.Schedule('PutItem', request, _OnPutItem) + + def DeleteItem(self, table, key, callback, expected=None, return_values=None): + assert not self._read_only, 'Received "DeleteItem" request on read-only database' + + table_def = self._schema.GetTable(table) + + def _OnDeleteItem(response): + callback(DeleteResult(return_values=self._FromDynamoAttributes(table_def, response.get('Attributes', None)), + write_units=response['ConsumedCapacityUnits'])) + + request = self._GetBaseRequest(table_def, key) + if expected is not None: + request['Expected'] = self._ToDynamoExpected(table_def, expected) + if return_values is not None: + request['ReturnValues'] = return_values + self._scheduler.Schedule('DeleteItem', request, _OnDeleteItem) + + def UpdateItem(self, table, key, callback, attributes, expected=None, + return_values=None): + assert not self._read_only, 'Received "UpdateItem" request on read-only database' + + table_def = self._schema.GetTable(table) + + def _OnUpdateItem(response): + callback(UpdateResult(return_values=self._FromDynamoAttributes(table_def, response.get('Attributes', None)), + write_units=response['ConsumedCapacityUnits'])) + + request = self._GetBaseRequest(table_def, key) + request['AttributeUpdates'] = self._ToDynamoAttributeUpdates(table_def, attributes) + if expected is not None: + request['Expected'] = self._ToDynamoExpected(table_def, expected) + if return_values is not None: + request['ReturnValues'] = return_values + self._scheduler.Schedule('UpdateItem', request, _OnUpdateItem) + + def Query(self, table, hash_key, range_operator, callback, attributes, + limit=None, consistent_read=False, count=False, + scan_forward=True, excl_start_key=None): + table_def = self._schema.GetTable(table) + + def _OnQuery(response): + callback(QueryResult(count=response['Count'], + items=[self._FromDynamoAttributes(table_def, item) for item in response.get('Items', [])], + last_key=self._FromDynamoKey(table_def, response.get('LastEvaluatedKey', None)), + read_units=response['ConsumedCapacityUnits'])) + + request = {'TableName': table_def.name_in_db} + request['HashKeyValue'] = self._ToDynamoValue(table_def.hash_key_col, hash_key) + if range_operator is not None: + request['RangeKeyCondition'] = { + 'AttributeValueList': [self._ToDynamoValue(table_def.range_key_col, rv) \ + for rv in range_operator.key], + 'ComparisonOperator': range_operator.op} + if attributes is not None: + request['AttributesToGet'] = attributes + if limit is not None: + request['Limit'] = limit + request['ConsistentRead'] = consistent_read + request['Count'] = count + request['ScanIndexForward'] = scan_forward + if excl_start_key is not None: + request['ExclusiveStartKey'] = self._ToDynamoKey(table_def, excl_start_key) + + self._scheduler.Schedule('Query', request, _OnQuery) + + def Scan(self, table, callback, attributes, limit=None, + excl_start_key=None, scan_filter=None): + table_def = self._schema.GetTable(table) + + def _OnScan(response): + callback(ScanResult(count=response['Count'], + items=[self._FromDynamoAttributes(table_def, item) for item in response['Items']], + last_key=self._FromDynamoKey(table_def, response.get('LastEvaluatedKey', None)), + read_units=response['ConsumedCapacityUnits'])) + + request = {'TableName': table_def.name_in_db} + if attributes is not None: + request['AttributesToGet'] = attributes + if limit is not None: + request['Limit'] = limit + #request['Count'] = count + if scan_filter is not None: + request['ScanFilter'] = dict() + for k, sf in scan_filter.items(): + col_def = table_def.GetColumnByKey(k) + request['ScanFilter'][k] = { + 'AttributeValueList': [self._ToDynamoValue(col_def, v) for v in sf.value], + 'ComparisonOperator': sf.op} + if excl_start_key is not None: + request['ExclusiveStartKey'] = self._ToDynamoKey(table_def, excl_start_key) + + self._scheduler.Schedule('Scan', request, _OnScan) + + def AddTimeout(self, deadline_secs, callback): + """Invokes the specified callback after 'deadline_secs'.""" + return ioloop.IOLoop.current().add_timeout(time.time() + deadline_secs, callback) + + def AddAbsoluteTimeout(self, abs_timeout, callback): + """Invokes the specified callback at time 'abs_timeout'.""" + return ioloop.IOLoop.current().add_timeout(abs_timeout, callback) + + def RemoveTimeout(self, timeout): + """Removes an existing timeout.""" + ioloop.IOLoopcurrent().remove_timeout(timeout) + + def _GetBaseRequest(self, table_def, key): + """Creates the base request structure for accessing a DynamoDB table + by key. + """ + return {'TableName': table_def.name_in_db, 'Key': self._ToDynamoKey(table_def, key)} + + def _FromDynamoKey(self, table_def, dyn_key): + """Converts a DynamoDB key into a DBKey named tuple, using the value + types defined in the table key definition. + """ + if dyn_key is None: + return None + value_type, value = dyn_key['HashKeyElement'].items()[0] + hash_key = self._FromDynamoValue(table_def.hash_key_col, value_type, value) + if table_def.range_key_col: + value_type, value = dyn_key['RangeKeyElement'].items()[0] + range_key = self._FromDynamoValue(table_def.range_key_col, value_type, value) + else: + range_key = None + return DBKey(hash_key, range_key) + + def _ToDynamoKey(self, table_def, key): + """Converts from a DBKey named tuple into a DynamoDB key.""" + dyn_key = {'HashKeyElement': self._ToDynamoValue(table_def.hash_key_col, key.hash_key)} + if key.range_key is not None: + assert table_def.range_key_col + dyn_key['RangeKeyElement'] = self._ToDynamoValue(table_def.range_key_col, key.range_key) + return dyn_key + + def _FromDynamoAttributes(self, table_def, dyn_attrs): + """Converts attributes as reported by DynamoDB into a dictionary + of key/value pairs. This verifies at each step that the value + types are in agreement, and converts from a list to a set for value + types 'SS' and 'NS'. + """ + if dyn_attrs is None: + return None + attrs = dict() + for k, v in dyn_attrs.items(): + value_type, value = v.items()[0] + attrs[k] = self._FromDynamoValue(table_def.GetColumnByKey(k), value_type, value) + return attrs + + def _ToDynamoAttributes(self, table_def, attrs): + """Converts attributes from schema datamodel to a dictionary + appropriate for use with DynamoDB JSON request protocol. + """ + dyn_attrs = dict() + for k, v in attrs.items(): + dyn_attrs[k] = self._ToDynamoValue(table_def.GetColumnByKey(k), v) + return dyn_attrs + + def _ToDynamoAttributeUpdates(self, table_def, updates): + """Converts attribute updates from schema datamodel to a + dictionary appropriate for use with DynamoDB JSON request + protocol. + """ + dyn_updates = dict() + for k, v in updates.items(): + dyn_updates[k] = {'Action': v.action} + if v.value is not None: + dyn_updates[k]['Value'] = self._ToDynamoValue(table_def.GetColumnByKey(k), v.value) + return dyn_updates + + def _ToDynamoExpected(self, table_def, expected): + """Converts expected values from schema datamodel to a dictionary + appropriate for use with DynamoDB JSON request protocol. If the + value of an expected key is a boolean, it must be False, and is + meant to specify that the attribute must not exist. + """ + dyn_exp = dict() + for k, v in expected.items(): + if isinstance(v, bool): + assert not v, 'if specifying a bool for an expected value, must be False' + dyn_exp[k] = {'Exists': False} + else: + dyn_exp[k] = {'Value': self._ToDynamoValue(table_def.GetColumnByKey(k), v)} + return dyn_exp + + def _FromDynamoValue(self, col_def, dyn_type, dyn_value): + """Converts a dynamo value to a python data structure for use with + viewfinder schema. + """ + assert col_def.value_type == dyn_type, '%s != %s' % (col_def.value_type, dyn_type) + if col_def.value_type == 'N': + return ConvertToNumber(dyn_value) + elif col_def.value_type == 'NS': + return set([ConvertToNumber(dv) for dv in dyn_value]) + elif col_def.value_type == 'SS': + return set([dv for dv in dyn_value]) + else: + return dyn_value + + def _ToDynamoValue(self, col_def, v): + """Converts a value to a representation appropriate for passing as a + JSON-encoded value to DynamoDB. + """ + if col_def.value_type == 'N': + return {col_def.value_type: ConvertToString(v)} + elif col_def.value_type == 'NS': + return {col_def.value_type: [ConvertToString(v_el) for v_el in v]} + elif col_def.value_type == 'SS': + return {col_def.value_type: [v_el for v_el in v]} + else: + assert col_def.value_type == 'S', col_def.value_type + return {col_def.value_type: v} + + def _GetTableSchema(self, name_in_db, desc): + """Builds a table schema namedtuple from a create or delete table request. + """ + assert desc['TableName'] == name_in_db, '%s != %s' % (desc['TableName'], name_in_db) + + def _GetDBKeySchema(key): + if 'KeySchema' in desc and key in desc['KeySchema']: + return DBKeySchema(name=desc['KeySchema'][key]['AttributeName'], + value_type=desc['KeySchema'][key]['AttributeType']) + else: + return None + + return TableSchema(create_time=desc.get('CreationDateTime', 0), + hash_key_schema=_GetDBKeySchema('HashKeyElement'), + range_key_schema=_GetDBKeySchema('RangeKeyElement'), + read_units=desc['ProvisionedThroughput']['ReadCapacityUnits'], + write_units=desc['ProvisionedThroughput']['WriteCapacityUnits'], + status=desc['TableStatus']) diff --git a/backend/db/episode.py b/backend/db/episode.py new file mode 100644 index 0000000..c8d8e2a --- /dev/null +++ b/backend/db/episode.py @@ -0,0 +1,191 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder episode. + +Episodes encompass a collection of photos separated by enough space +and/or time from previous or subsequent photos. + +Episode ids are constructed from 32 bits of time, a variable-length- +encoded integer device id and a variable-length-encoded unique id from +the device. The final value is base64-hex encoded. They sort +lexicographically by timestamp (reverse ordered so the most recent +episodes are listed first in a query). + +Photos are added to episodes via posts. A post is a composite-key +relation between an episode-id and a photo-id. + + Episode: a collection of photos contiguous in spacetime +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging + +from functools import partial +from tornado import gen +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import InvalidRequestError, PermissionError, LimitExceededError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.asset_id import IdPrefix, ConstructTimestampAssetId +from viewfinder.backend.db.asset_id import DeconstructTimestampAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post +from viewfinder.backend.db.user import User +from viewfinder.backend.db.user_photo import UserPhoto +from viewfinder.backend.db.user_post import UserPost +from viewfinder.backend.db.versions import Version +from viewfinder.backend.db.viewpoint import Viewpoint + + +@DBObject.map_table_attributes +class Episode(DBHashObject): + """Viewfinder episode data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.EPISODE) + + def __init__(self, episode_id=None): + super(Episode, self).__init__() + self.episode_id = episode_id + + @classmethod + def ConstructEpisodeId(cls, timestamp, device_id, uniquifier): + """Returns an episode id constructed from component parts. Episodes + sort from newest to oldest. See "ConstructTimestampAssetId" for + details of the encoding. + """ + return ConstructTimestampAssetId(IdPrefix.Episode, timestamp, device_id, uniquifier) + + @classmethod + def DeconstructEpisodeId(cls, episode_id): + """Returns the components of an episode id: timestamp, device_id, and + uniquifier. + """ + return DeconstructTimestampAssetId(IdPrefix.Episode, episode_id) + + @classmethod + @gen.coroutine + def VerifyEpisodeId(cls, client, user_id, device_id, episode_id): + """Ensures that a client-provided episode id is valid according + to the rules specified in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Episode, episode_id, has_timestamp=True) + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **ep_dict): + """Creates the episode specified by 'ep_dict'. The caller is responsible for checking + permission to do this, as well as ensuring that the episode does not yet exist (or is + just being identically rewritten). + + Returns: The created episode. + """ + assert 'episode_id' in ep_dict and 'user_id' in ep_dict and 'viewpoint_id' in ep_dict, ep_dict + assert 'timestamp' in ep_dict, 'timestamp attribute required in episode: "%s"' % ep_dict + assert 'publish_timestamp' in ep_dict, 'publish_timestamp attribute required in episode: "%s"' % ep_dict + + episode = Episode.CreateFromKeywords(**ep_dict) + yield gen.Task(episode.Update, client) + raise gen.Return(episode) + + @gen.coroutine + def UpdateExisting(self, client, **ep_dict): + """Updates an existing episode.""" + assert 'publish_timestamp' not in ep_dict and 'parent_ep_id' not in ep_dict, ep_dict + self.UpdateFromKeywords(**ep_dict) + yield gen.Task(self.Update, client) + + @classmethod + @gen.coroutine + def QueryIfVisible(cls, client, user_id, episode_id, must_exist=True, consistent_read=False): + """If the user has viewing rights to the specified episode, returns that episode, otherwise + returns None. The user has viewing rights if the user is a follower of the episode's + viewpoint. If must_exist is true and the episode does not exist, raises an InvalidRequest + exception. + """ + episode = yield gen.Task(Episode.Query, + client, + episode_id, + None, + must_exist=False, + consistent_read=consistent_read) + + if episode is None: + if must_exist == True: + raise InvalidRequestError('Episode "%s" does not exist.' % episode_id) + else: + follower = yield gen.Task(Follower.Query, + client, + user_id, + episode.viewpoint_id, + col_names=None, + must_exist=False) + if follower is None or not follower.CanViewContent(): + raise gen.Return(None) + + raise gen.Return(episode) + + @classmethod + def QueryPosts(cls, client, episode_id, user_id, callback, + limit=None, excl_start_key=None, base_results=None): + """Queries posts (up to 'limit' total) for the specified + 'episode_id', viewable by 'user_id'. The query is for posts starting + with (but excluding) 'excl_start_key'. The photo metadata for each + post relation are in turn queried and the post and photo metadata + are combined into a single dict. The callback is invoked with the + array of combined post/photo metadata, and the last queried post + sort-key. + + The 'base_results' argument allows this method to be re-entrant. + 'limit' can be satisfied completely if a user is querying an + episode they own with nothing archived or deleted. However, in cases + where an episode hasn't been fully shared, or has many photos archived + or deleted by the requesting user, QueryPosts needs to be re-invoked + possibly many times to query 'limit' posts or reach the end of the + episode. + """ + def _OnQueryMetadata(posts, results): + """Constructs the photo metadata to return. The "check_label" argument + is used to determine whether to use the old permissions model or the + new one. If "check_label" is true, then only return a photo if a label + is present. Otherwise, the photo is part of an episode created by + the new sharing functionality, and the user automatically has access + to all photos in that episode. + """ + ph_dicts = base_results or [] + for post, (photo, user_post) in zip(posts, results): + ph_dict = photo._asdict() + labels = post.labels.combine() + if user_post is not None: + labels = labels.union(user_post.labels.combine()) + if len(labels) > 0: + ph_dict['labels'] = list(labels) + ph_dicts.append(ph_dict) + + last_key = posts[-1].photo_id if len(posts) > 0 else None + if last_key is not None and len(ph_dicts) < limit: + Episode.QueryPosts(client, episode_id, user_id, callback, limit=limit, + excl_start_key=last_key, base_results=ph_dicts) + else: + callback((ph_dicts, last_key)) + + def _OnQueryPosts(posts): + with util.ArrayBarrier(partial(_OnQueryMetadata, posts)) as b: + for post in posts: + with util.ArrayBarrier(b.Callback()) as metadata_b: + post_id = Post.ConstructPostId(post.episode_id, post.photo_id) + Photo.Query(client, hash_key=post.photo_id, col_names=None, + callback=metadata_b.Callback()) + UserPost.Query(client, hash_key=user_id, range_key=post_id, + col_names=None, callback=metadata_b.Callback(), must_exist=False) + + # Query the posts with limit & excl_start_key. + Post.RangeQuery(client, hash_key=episode_id, range_desc=None, limit=limit, + col_names=None, callback=_OnQueryPosts, excl_start_key=excl_start_key) diff --git a/backend/db/followed.py b/backend/db/followed.py new file mode 100644 index 0000000..812ade3 --- /dev/null +++ b/backend/db/followed.py @@ -0,0 +1,81 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Followed relation. + +The Followed relation is basically a secondary index over the +"last_updated" attribute of the Viewpoint table. Viewpoints are +ordered according to the time of last update, rather than by +viewpoint id. However, the ordering is not perfectly maintained. +Viewpoints that were last updated on the same day are grouped +together, with the ordering within the group undefined. This +enables "query_followed" to return viewpoints in rough order, but +without paying a high cost for keeping the index maintained. + + Followed: sorts viewpoints in reverse order of last update. +""" + +__authors__ = ['andy@emailscrubbed.com (Andy Kimball)'] + +from tornado import gen +from viewfinder.backend.base import constants, util +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class Followed(DBRangeObject): + """Viewfinder followed data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.FOLLOWED) + + def __init__(self, user_id=None, sort_key=None): + super(Followed, self).__init__() + self.user_id = user_id + self.sort_key = sort_key + + @classmethod + def CreateSortKey(cls, viewpoint_id, timestamp): + """Creates a "sort_key" value, which is a concatenation of the timestamp + (truncated to day boundary) and the viewpoint id. + """ + # Reverse the timestamp so that Viewpoints sort with the latest updated first. + prefix = util.CreateSortKeyPrefix(Followed._TruncateToDay(timestamp), randomness=False, reverse=True) + return prefix + viewpoint_id + + @classmethod + @gen.engine + def UpdateDateUpdated(cls, client, user_id, viewpoint_id, old_timestamp, new_timestamp, callback): + """Inserts a new followed record with date_updated set to the truncated "new_timestamp", + and then deletes the followed record for "old_timestamp". A simple update is not possible + because the "date_updated" attribute is part of the primary key. Optimize by not updating + if the old and new "date_updated" values are the same. + """ + # Always ratchet the timestamp -- never update to an older timestamp. + assert new_timestamp is not None, (user_id, viewpoint_id) + if old_timestamp is None or old_timestamp < new_timestamp: + old_date_updated = Followed._TruncateToDay(old_timestamp) + new_date_updated = Followed._TruncateToDay(new_timestamp) + + # Only update (and possibly delete) if old and new values are not the same. + if old_date_updated != new_date_updated: + # Insert the new followed record. + followed = Followed(user_id, Followed.CreateSortKey(viewpoint_id, new_date_updated)) + followed.date_updated = new_date_updated + followed.viewpoint_id = viewpoint_id + yield gen.Task(followed.Update, client) + + # Delete the previous followed record, if it exists. + if old_date_updated is not None: + followed = Followed(user_id, Followed.CreateSortKey(viewpoint_id, old_date_updated)) + yield gen.Task(followed.Delete, client) + + callback() + + @classmethod + def _TruncateToDay(cls, timestamp): + """Truncate timestamp to day boundary.""" + if timestamp is None: + return None + return (timestamp // constants.SECONDS_PER_DAY) * constants.SECONDS_PER_DAY diff --git a/backend/db/follower.py b/backend/db/follower.py new file mode 100644 index 0000000..78b063e --- /dev/null +++ b/backend/db/follower.py @@ -0,0 +1,199 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Follower relation. + +A Follower object defines a relation between a user and a viewpoint. +If a user is a follower of a viewpoint, episodes added to the +viewpoint are shared with the user. + +Each follower contains a set of labels which describe properties and +permissions. See the descriptions for each label in the header of the +Follower class. + + Follower: defines relation between a user and a viewpoint. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging + +from tornado import gen + +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import PermissionError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.range_base import DBRangeObject + +@DBObject.map_table_attributes +class Follower(DBRangeObject): + """Viewfinder follower data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.FOLLOWER) + + ADMIN = 'admin' + """The follower can change the permissions of any other follower in the viewpoint.""" + + CONTRIBUTE = 'contribute' + """The follower can change viewpoint metadata, add new followers, and contribute content to + the viewpoint. + """ + + PERSONAL = 'personal' + """Photos in this viewpoint should be shown in the summary and day views of the follower.""" + + REMOVED = 'removed' + """This viewpoint has been removed from the follower's inbox and is no longer accessible. + Quota should not be counted against the account. If all followers have marked a viewpoint + 'removed', then the viewpoint's resources may be garbage collected. Followers with this + label should not be able to add or view any content in the viewpoint. Attempts to modify + content should fail with a permission error. Attempts to view content should return empty + content. + """ + + MUTED = 'muted' + """Alerts for the associated viewpoint will not be sent to this follower.""" + + HIDDEN = 'hidden' + """Any "trap doors" to this viewpoint should *not* be shown in the summary view of the + follower. + """ + + AUTOSAVE = 'autosave' + """Any time photos are added to the viewpoint, they are automatically saved to this follower's + default viewpoint. + """ + + UNREVIVABLE = 'unrevivable' + """This label should be set only if REMOVED is also set. If it is set, then the follower + cannot be revived when activity occurs in the followed viewpoint. Only another follower + re-adding the user to the viewpoint will restore access. + """ + + PERMISSION_LABELS = [ADMIN, CONTRIBUTE] + """Labels that specify what the follower is allowed to do in the viewpoint.""" + + UNSETTABLE_LABLES = PERMISSION_LABELS + [REMOVED, UNREVIVABLE] + """Labels that should never be directly set by an end user, but are instead indirectly set + by various operations. + """ + + ALL_LABELS = PERMISSION_LABELS + [PERSONAL, REMOVED, HIDDEN, MUTED, UNREVIVABLE, AUTOSAVE] + """Viewpoint permissions and modifiers that apply to the follower.""" + + def __init__(self, user_id=None, viewpoint_id=None): + super(Follower, self).__init__() + self.user_id = user_id + self.viewpoint_id = viewpoint_id + + def CanViewContent(self): + """Returns true if the follower has not been REMOVED. REMOVED followers are not allowed + to view viewpoint content. + """ + return Follower.REMOVED not in self.labels + + def CanAdminister(self): + """Returns true if the follower has the ADMIN permission and hasn't been REMOVED.""" + return Follower.ADMIN in self.labels and Follower.REMOVED not in self.labels + + def CanContribute(self): + """Returns true if the follower has the CONTRIBUTE permission and hasn't been REMOVED.""" + return Follower.CONTRIBUTE in self.labels and Follower.REMOVED not in self.labels + + def IsRemoved(self): + """Returns true if the follower has the Follower.REMOVED label.""" + return Follower.REMOVED in self.labels + + def IsMuted(self): + """Returns true if alerts should be suppressed for this follower.""" + return Follower.MUTED in self.labels + + def IsUnrevivable(self): + """Returns true if the follower cannot be revived when activity on the followed viewpoint + occurs. + """ + return Follower.UNREVIVABLE in self.labels + + def ShouldAutoSave(self): + """Returns true if photos added to the viewpoint should be automatically saved to this + follower's default viewpoint. + """ + return Follower.AUTOSAVE in self.labels + + def MakeMetadataDict(self): + """Projects all follower attributes that the follower himself can see.""" + foll_dict = {'follower_id': self.user_id} + util.SetIfNotNone(foll_dict, 'adding_user_id', self.adding_user_id) + util.SetIfNotNone(foll_dict, 'viewed_seq', self.viewed_seq) + if self.labels is not None: + # Normalize labels property for easier testing. + foll_dict['labels'] = sorted(self.labels) + return foll_dict + + def MakeFriendMetadataDict(self): + """Projects a subset of the follower attributes that should be provided to another user + that is on the same viewpoint as this follower. + """ + foll_dict = {'follower_id': self.user_id} + util.SetIfNotNone(foll_dict, 'adding_user_id', self.adding_user_id) + util.SetIfNotNone(foll_dict, 'follower_timestamp', self.timestamp) + if self.IsUnrevivable(): + # Only project labels if the follower has left the viewpoint entirely. + foll_dict['labels'] = [Follower.REMOVED, Follower.UNREVIVABLE] + return foll_dict + + def SetLabels(self, new_labels): + """Sets the labels attribute on the follower. This must be done with care in order to + avoid security bugs such as allowing users to give themselves admin permissions, or + allowing users to accidentally remove their right to see the viewpoint, or allowing a + viewpoint to be removed without updating quota. + + TODO(Andy): Eventually we'll want more full-featured control over + permissions. + """ + new_labels = set(new_labels) + new_unsettable_labels = new_labels.intersection(Follower.UNSETTABLE_LABLES) + existing_labels = set(self.labels) + existing_unsettable_labels = existing_labels.intersection(Follower.UNSETTABLE_LABLES) + if new_unsettable_labels != existing_unsettable_labels: + raise PermissionError('Permission and removed labels cannot be updated on the follower.') + self.labels = new_labels + + @gen.coroutine + def RemoveViewpoint(self, client, allow_revive=True): + """Removes a viewpoint from a user's inbox, and its content will become inaccessible to + this follower. If "allow_revive" is true, then the viewpoint will automatically be + "revived" when there is new activity by other followers that have not removed it. + + Adds the REMOVED label to this follower object and updates the db. Caller should have already + checked permissions to do this. Adds the UNREVIVABLE label if "allow_revive" is false. + """ + if not self.IsRemoved(): + # If not removed, then UNREVIVABLE flag should never be set. + assert not self.IsUnrevivable(), self + + # Add follower label and persist the change. + self.labels.add(Follower.REMOVED) + + if not allow_revive: + self.labels.add(Follower.UNREVIVABLE) + + yield gen.Task(self.Update, client) + + @classmethod + @gen.coroutine + def ReviveRemovedFollowers(cls, client, followers): + """Removes the REMOVED labels from any followers which are not marked as UNREVIVABLE, and + updates those records in the DB. + """ + tasks = [] + for follower in followers: + if follower.IsRemoved() and not follower.IsUnrevivable(): + follower.labels.remove(Follower.REMOVED) + tasks.append(gen.Task(follower.Update, client)) + + yield tasks diff --git a/backend/db/friend.py b/backend/db/friend.py new file mode 100644 index 0000000..4ecbae0 --- /dev/null +++ b/backend/db/friend.py @@ -0,0 +1,173 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Friend relation. + +Viewfinder friends define a relationship between two users predicated on confirmation of photo +sharing permission. Each friend has an associated 'status', which can be: + + - 'friend': user has been marked as a friend; however, that user may not have the reverse + friendship object. + - 'muted': a friend who has attained special status as an unwanted irritant. Content shared + from these friends is not shown, though still received and can be retrieved. + - 'blocked': a friend who has attained special status as an unwanted irritant. These users will + not show up in suggestions lists and cannot be contacted for sharing. + +Friends are different than contacts. Contacts are the full spectrum of social connections. A +contact doesn't become a viewfinder friend until a share has been completed. + +NOTE: Next comment is outdated, but we may re-enable something similar in future. +The 'colocated_shares', 'total_shares', 'last_colocated' and 'last_share' values are used to +quantify the strength of the sharing relationship. Each time the users in a friend relationship +are co-located, 'colocated_shares' is decayed based on 'last_colocated' and the current time +and updated either with a +1 if the sharing occurs or a -1 if not. 'total_shares' is similarly +updated, though not just when the users are co-located, but on every share that a user initiates. + + Friend: viewfinder friend information +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging +import math + +from functools import partial +from tornado import gen +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import NotFoundError +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.op.notification_manager import NotificationManager + + +@DBObject.map_table_attributes +class Friend(DBRangeObject): + """Viewfinder friend data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.FRIEND) + + FRIEND = 'friend' + MUTED = 'muted' + BLOCKED = 'blocked' + + FRIEND_ATTRIBUTES = set(['nickname']) + """Subset of friend attributes that should be projected to the user.""" + + _SHARE_HALF_LIFE = 60 * 60 * 24 * 30 # 1 month + + def __init__(self, user_id=None, friend_id=None): + super(Friend, self).__init__() + self.user_id = user_id + self.friend_id = friend_id + self.status = Friend.FRIEND + + def IsBlocked(self): + """Returns true if the "friend" identified by self.friend_id is blocked.""" + return self.status == Friend.BLOCKED + + def DecayShares(self, timestamp): + """Decays 'total_shares' and 'colocated_shares' based on 'timestamp'. Updates 'last_share' + and 'last_colocated' to 'timestamp'. + """ + def _ComputeDecay(shares, last_time): + if last_time is None: + assert shares is None, shares + return 0 + decay = math.exp(-math.log(2) * (timestamp - last_time) / + Friend._SHARE_HALF_LIFE) + return shares * decay + + self.total_shares = _ComputeDecay(self.total_shares, self.last_share) + self.last_share = timestamp + self.colocated_shares = _ComputeDecay(self.colocated_shares, self.last_colocated) + self.last_colocated = timestamp + + def IncrementShares(self, timestamp, shared, colocated): + """Decays and updates 'total_shares' and 'last_share' based on whether sharing occurred + ('shared'==True). If 'colocated', the 'colocated_shares' and 'last_colocated' are updated + similarly. + """ + self.DecayShares(timestamp) + self.total_shares += (1.0 if shared else -1.0) + if colocated: + self.colocated_shares += (1.0 if shared else -1.0) + + @classmethod + @gen.engine + def MakeFriends(cls, client, user_id, friend_id, callback): + """Creates a bi-directional friendship between user_id and friend_id if it does not already + exist. Invokes the callback with the pair of friendship objects: + [(user_id=>friend_id), (friend_id=>user_id)] + """ + from viewfinder.backend.db.user import User + + # Determine whether one or both sides of the friendship are missing. + forward_friend, reverse_friend = \ + yield [gen.Task(Friend.Query, client, user_id, friend_id, None, must_exist=False), + gen.Task(Friend.Query, client, friend_id, user_id, None, must_exist=False)] + + # Make sure that both sides of the friendship have been created. + if forward_friend is None: + forward_friend = Friend.CreateFromKeywords(user_id=user_id, friend_id=friend_id, status=Friend.FRIEND) + yield gen.Task(forward_friend.Update, client) + + if reverse_friend is None: + reverse_friend = Friend.CreateFromKeywords(user_id=friend_id, friend_id=user_id, status=Friend.FRIEND) + yield gen.Task(reverse_friend.Update, client) + + callback((forward_friend, reverse_friend)) + + @classmethod + @gen.engine + def MakeFriendsWithGroup(cls, client, user_ids, callback): + """Creates bi-directional friendships between all the specified users. Each user will be + friends with every other user. + """ + yield [gen.Task(Friend.MakeFriends, client, user_id, friend_id) + for index, user_id in enumerate(user_ids) + for friend_id in user_ids[index + 1:] + if user_id != friend_id] + callback() + + @classmethod + @gen.engine + def MakeFriendAndUpdate(cls, client, user_id, friend_dict, callback): + """Ensures that the given user has at least a one-way friend relationship with the given + friend. Updates the friend relationship attributes with those given in "friend_dict". + """ + from viewfinder.backend.db.user import User + + friend = yield gen.Task(Friend.Query, client, user_id, friend_dict['user_id'], None, must_exist=False) + + if friend is None: + # Ensure that the friend exists as user in the system. + friend_user = yield gen.Task(User.Query, client, friend_dict['user_id'], None, must_exist=False) + if friend_user is None: + raise NotFoundError('User %d does not exist.' % friend_dict['user_id']) + + # Create a one-way friend relationship from the calling user to the friend user. + friend = Friend.CreateFromKeywords(user_id=user_id, friend_id=friend_dict['user_id'], status=Friend.FRIEND) + + # Update all given attributes. + assert friend_dict['user_id'] == friend.friend_id, (friend_dict, friend) + for key, value in friend_dict.iteritems(): + if key != 'user_id': + assert key in Friend.FRIEND_ATTRIBUTES, friend_dict + setattr(friend, key, value) + + yield gen.Task(friend.Update, client) + callback() + + @classmethod + @gen.engine + def UpdateOperation(cls, client, callback, user_id, friend): + """Updates friend metadata for the relationship between the given user and friend.""" + # Update the metadata. + yield gen.Task(Friend.MakeFriendAndUpdate, client, user_id, friend) + + # Send notifications to all the calling user's devices. + yield NotificationManager.NotifyUpdateFriend(client, friend) + + callback() diff --git a/backend/db/guess.py b/backend/db/guess.py new file mode 100644 index 0000000..b14e69e --- /dev/null +++ b/backend/db/guess.py @@ -0,0 +1,68 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Tracks the number of incorrect attempts that have been made to guess a protected secret. + +It is important to track incorrect guesses in order to mitigate brute-force attacks. Each +time a guess is made (correct or incorrect), call Guess.Report, which will track and limit +the number of incorrect guesses. + +See the header for the GUESS table in vf_schema.py for additional details about the table. +""" + +__authors__ = ['andy@emailscrubbed.com (Andy Kimball)'] + +from tornado import gen +from viewfinder.backend.base import constants, util +from viewfinder.backend.base.exceptions import TooManyGuessesError +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject + + +@DBObject.map_table_attributes +class Guess(DBHashObject): + """Viewfinder guess data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.GUESS) + _guesses_key = _table.GetColumn('guesses').key + + def __init__(self, guess_id=None): + super(Guess, self).__init__() + self.guess_id = guess_id + + @classmethod + def ConstructGuessId(cls, type, id): + """Construct a guess id of the form :.""" + return "%s:%s" % (type, id) + + @classmethod + @gen.coroutine + def CheckGuessLimit(cls, client, guess_id, max_guesses): + """Returns false if the number of incorrect guesses has already exceeded "max_guesses".""" + guess = yield gen.Task(Guess.Query, client, guess_id, None, must_exist=False) + + # If guess record is expired, ignore it -- it will be re-created in that case. + now = util.GetCurrentTimestamp() + if guess is not None and now >= guess.expires: + guess = None + + raise gen.Return(guess is None or guess.guesses < max_guesses) + + @classmethod + @gen.coroutine + def ReportIncorrectGuess(cls, client, guess_id): + """Records an incorrect guess attempt by incrementing the guesses count.""" + guess = yield gen.Task(Guess.Query, client, guess_id, None, must_exist=False) + + # Increment the incorrect guess count. + now = util.GetCurrentTimestamp() + if guess is not None and now < guess.expires: + guess.guesses += 1 + else: + guess = Guess(guess_id) + guess.expires = now + constants.SECONDS_PER_DAY + guess.guesses = 1 + + yield gen.Task(guess.Update, client) diff --git a/backend/db/hash_base.py b/backend/db/hash_base.py new file mode 100644 index 0000000..a828f4c --- /dev/null +++ b/backend/db/hash_base.py @@ -0,0 +1,87 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Base object for database objects from tables with a simple hash key. + +Sub classes must implement GetKey, _GetIndexKey, and _ParseIndexKey. +See the comments below for a description of each method. + + DBHashObject: base class of all hash-key data objects +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging + +from tornado.concurrent import return_future + +from viewfinder.backend.base import util +from viewfinder.backend.db import db_client, schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.versions import Version + + +class DBHashObject(DBObject): + """Base class for items from tables with a hash key. + """ + __slots__ = [] + + def __init__(self, columns=None): + super(DBHashObject, self).__init__(columns=columns) + + @classmethod + @return_future + def Query(cls, client, hash_key, col_names, callback, + must_exist=True, consistent_read=False): + """Queries a object by primary hash key.""" + cls.KeyQuery(client, key=db_client.DBKey(hash_key=hash_key, range_key=None), + col_names=col_names, callback=callback, must_exist=must_exist, + consistent_read=consistent_read) + + @classmethod + def Allocate(cls, client, callback): + """Allocates a new primary key via the id_allocator table. Invokes + the provided callback with new object. + """ + assert cls._allocator, 'class has no id allocator declared' + def _OnAllocate(obj_id): + o = cls(obj_id) + o._columns[schema.Table.VERSION_COLUMN.name].Set(Version.GetCurrentVersion()) + callback(o) + cls._allocator.NextId(client, _OnAllocate) + + def GetKey(self): + """Returns the object's primary hash key.""" + return db_client.DBKey(hash_key=self._columns[self._table.hash_key_col.name].Get(), + range_key=None) + + @classmethod + def _MakeIndexKey(cls, db_key): + """Creates an indexing key from the provided object key. This is + symmetric with _ParseIndexKey. All index keys are stored as strings, + so we get a string representation here in case the hash key column + is a number. + """ + val = db_key.hash_key + if cls._table.hash_key_col.value_type == 'N': + assert isinstance(val, (int, long)), 'primary hash key not of type int or long' + val = str(val) + return val + + @classmethod + def _ParseIndexKey(cls, index_key): + """Returns the object's key by parsing the index key. This is + symmetric with _MakeIndexKey, and is used to extract the actual + object key from results of index queries. By default, returns the + unadulterated index_key. + + Because all keys are stored in the index table as strings, if the + hash key column type is a number, convert here from a string to a + number. + """ + if cls._table.hash_key_col.value_type == 'N': + index_key = int(index_key) + return db_client.DBKey(hash_key=index_key, range_key=None) + + @classmethod + def _GetIndexedObjectClass(cls): + return cls diff --git a/backend/db/health_report.py b/backend/db/health_report.py new file mode 100644 index 0000000..68513ca --- /dev/null +++ b/backend/db/health_report.py @@ -0,0 +1,300 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder health report. A report represents a guess at the overall health +of the server at a given time. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import logging +import math +import time +from collections import namedtuple +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import DBConditionalCheckFailedError +from viewfinder.backend.base.counters import counters +from viewfinder.backend.db import db_client, vf_schema, metric +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.op import user_op_manager + + +# Because alerts and warnings are a flat list, this token is used to indicate that an alert +# or warning applies to the entire cluster rather than a single machine. +CLUSTER_TOKEN = 'CLUSTER' + + +# Number of previous reports to consider when looking for trends in performance counters. +# Certain warnings may be escalated to alerts if they occur too often. +TREND_SAMPLE_SIZE = 9 + + +def ErrorCriteria(errors): + """Monitor the number of unexpected errors logged in the cluster. If more than five + errors have occurred on the cluster during this time period, post an alert. Posts a + warning if between one and four errors have occurred. + """ + ERROR_ALERT_THRESHOLD = 5 + alerts = [] + warnings = [] + + if errors['cluster_total'] > ERROR_ALERT_THRESHOLD: + alerts.append(CLUSTER_TOKEN) + elif errors['cluster_total'] > 0: + warnings.append(CLUSTER_TOKEN) + + return alerts, warnings + + +def RequestsFailedCriteria(request_rate, failed_rate): + """Monitor the rate of failed service requests on the server. The number of failed + requests will be compared to the total number of requests to determine if a warning + is warranted, either for the cluster or for any individual machine. + """ + alerts = [] + warnings = [] + + def _ComputeThreshold(x): + # Failure threshold is defined as the square root of the total request rate. This gives + # an appropriate threshold for both very low and very high numbers. For example, the + # threshold is approximately 30% of requests for 10 total requests, 10% for 100 and + # 3% for 1000. + return math.ceil(math.sqrt(x)) + + if failed_rate['cluster_total'] > _ComputeThreshold(request_rate['cluster_total']): + warnings.append(CLUSTER_TOKEN) + + for m, v in request_rate['machine_data'].iteritems(): + if failed_rate['machine_data'][m] > _ComputeThreshold(v): + warnings.append(m) + + return alerts, warnings + + +def OperationRetriesCriteria(operation_rate, retry_rate): + """Monitor the rate of operation retries on the server. The number of operation retries + will be compared to the total number of operations run. + """ + alerts = [] + warnings = [] + + def _ComputeThreshold(x): + # Failure threshold is defined as one-third of the square root of the total request rate. + # This gives an appropriate threshold for both very low and very high numbers. For example, + # the threshold is approximately 10% of requests for 10 total requests, 3% for 100 and + # 1% for 1000. + return math.ceil(math.sqrt(x)) / 3 + + if retry_rate['cluster_total'] > _ComputeThreshold(operation_rate['cluster_total']): + warnings.append(CLUSTER_TOKEN) + + for m, v in operation_rate['machine_data'].iteritems(): + if retry_rate['machine_data'][m] > _ComputeThreshold(v): + warnings.append(m) + + return alerts, warnings + + +def MissingMetricsCriteria(): + """This criteria is alerted if metrics data is completely missing at a timestamp. + This is a special criteria in that warnings are generated directly in the GetHealthReport + method, rather than in this criteria. + """ + return [], [] + + +class HealthCriteria(object): + """Class which encapsulates a single health criteria for a report. Each criteria + checks a set of counters at the report timestamp, returning a list of alerts and + warnings based on the timestamp. A criteria can also evaluate trends over the last + several health reports, which may result in a warning escalating to an alert. + + 'alert_description' is a description of the condition which would result in this criteria + raising an alert. + + 'counter_list' is a list of counters which the criteria needs to operate on. The values + for each counter at the report timestamp will be passed to the handler as parameters + in the order requested. + + 'handler' is a method which accepts the requested counter data and returns a list of + machines for which the criteria should alert, and a second list of machines for which + the criteria should warn. + + Finally, 'escalation_threshold' is an optional value which is used to escalate warnings + to alerts based on the value of previous reports. If the threshold is non-zero and + a warning has been present in a number of previous reports exceeding the threshold, + then the warning is escalated to an alert. + """ + def __init__(self, criteria_name, alert_description, handler, counter_list, escalation_threshold=0): + self.name = criteria_name + self.description = alert_description + self.counter_list = counter_list + self.handler = handler + self.escalation_threshold = escalation_threshold + + def InspectMetrics(self, metric_data, report): + """Passes the requested counter data for this criteria to the handler, and adds + the returned set of warnings and alerts to the report. + """ + alerts, warnings = self.handler(*[metric_data[c.name] for c in self.counter_list]) + for w in warnings: + report.warnings.add(self.name + ':' + w) + for a in alerts: + report.alerts.add(self.name + ':' + a) + + def InspectTrends(self, old_reports, new_report): + """If an escalation threshold is set for this criteria, inspects previous reports + and escalates warnings to alerts if they are present in a number of previous reports + exceeding the threshold. + """ + if self.escalation_threshold == 0: + return + + current_warnings = [w for w in new_report.warnings.combine() if w.startswith(self.name + ':')] + for w in current_warnings: + if len([r for r in old_reports if w in r.warnings]) > self.escalation_threshold: + new_report.alerts.add(w) + + @classmethod + def GetCriteriaList(cls): + """Gets the list of criteria for server health checks. This is implemented as a class + method to ensure that the static counters variable is completely loaded before accessing + it, rather than depending on python module loading order. + """ + + if hasattr(cls, '_criteria_list'): + return cls._criteria_list + + cls._criteria_list = [ + HealthCriteria('Errors', + 'Error threshold exceeded.', + ErrorCriteria, + [counters.viewfinder.errors.error], + 5), + HealthCriteria('ReqFail', + 'Failed Request threshold exceeded.', + RequestsFailedCriteria, + [counters.viewfinder.service.req_per_min, counters.viewfinder.service.fail_per_min], + 5), + HealthCriteria('OpRetries', + 'Operation retry threshold exceeded.', + OperationRetriesCriteria, + [counters.viewfinder.operation.ops_per_min, counters.viewfinder.operation.retries_per_min], + 5), + HealthCriteria('MissingMetrics', + 'Metrics collection failed.', + MissingMetricsCriteria, + [], + 3), + ] + return cls._criteria_list + + +@DBObject.map_table_attributes +class HealthReport(DBRangeObject): + """Class which describes the overall health of a metrics group at a given timestamp. Health + is determined based on the aggregated performance counters of the metrics group over a + period of time. + + A health report record is designed to be sparse if the group is healthy - only a list + of warnings and alerts is stored. If the cluster is completely healthy according to + the configured criteria, then the record will essentially be empty. + """ + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.HEALTH_REPORT) + + def __init__(self, group_key=None, timestamp=None): + super(HealthReport, self).__init__() + self.group_key = group_key + self.timestamp = timestamp + + @classmethod + def QueryTimespan(cls, client, group_key, start_time, end_time, callback, excl_start_key=None): + """ Performs a range query on the HealthReport table for the given group_key between + the given start and end time. An optional start key can be specified to resume + an earlier query which did not retrieve the full result set. + """ + HealthReport.RangeQuery(client, group_key, db_client.RangeOperator([start_time, end_time], 'BETWEEN'), + None, None, callback=callback, excl_start_key=excl_start_key) + + @classmethod + def GetHealthReport(cls, client, cluster_name, interval, timestamp, callback, counter_set=None, criteria=None): + """ Get a cluster health report for the given cluster and collection interval at the + given timestamp. The report will be generated if it is not already available in the database. + The given callback will be invoked with the report once it is retrieved or generated. + + A specific counter set can be provided if desired; by default, all counters registered globally + with the counters module will be used. + + The optional criteria parameter is intended for testing, but can be used to provide an optional + list of HealthCriteria objects used to generate the report. By default, the list provided by + HealthCriteria.GetCriteriaList() will be used. + """ + criteria = criteria or HealthCriteria.GetCriteriaList() + counter_set = counter_set or counters + group_key = metric.Metric.EncodeGroupKey(cluster_name, interval) + + # Calculate points in time relevant to this report. + newest_report_timestamp = timestamp - (timestamp % interval.length) + oldest_report_timestamp = newest_report_timestamp - (interval.length * TREND_SAMPLE_SIZE) + + def _OnCreateReport(old_reports, new_report): + _OnQueryPreviousReports(old_reports + [new_report]) + + def _OnFailCreateReport(type_, value_, traceback_): + if type_ is DBConditionalCheckFailedError: + # Another machine has already generated the new report, restart at _OnQueryReport. + _OnQueryReport(None) + else: + raise type_, value_, traceback_ + + def _OnAggregate(timestamp, reports, metrics): + new_report = HealthReport.CreateFromKeywords(group_key=group_key, timestamp=timestamp) + + if len(metrics.timestamps) == 0: + # No metrics collected for this period. Only the missing metrics criteria + # can be evaluated in this case. + new_report.warnings.add('MissingMetrics:' + CLUSTER_TOKEN) + for c in criteria: + c.InspectTrends(reports, new_report) + else: + # Pivot metrics data to time. + time_pivot = {counter: {'cluster_total': data.cluster_total[0][1], + 'cluster_avg': data.cluster_avg[0][1], + 'machine_data': {k : v[0][1] for k, v in data.machine_data.iteritems()}, + } for counter, data in metrics.counter_data.iteritems() + } + + for c in criteria: + c.InspectMetrics(time_pivot, new_report) + c.InspectTrends(reports, new_report) + + success_cb = partial(_OnCreateReport, reports, new_report) + with util.Barrier(success_cb, _OnFailCreateReport) as b: + new_report.Update(client, b.Callback(), replace=False) + + def _OnQueryPreviousReports(reports): + if (len(reports) == TREND_SAMPLE_SIZE + 1): + # The report for the requested timestamp is now available. + callback(reports[-1]) + return + + # Generate the first missing report chronologically. + next_timestamp = (reports[-1].timestamp + interval.length) if len(reports) > 0 else oldest_report_timestamp + metric.AggregatedMetric.CreateAggregateForTimespan(client, group_key, next_timestamp, next_timestamp, + counter_set, partial(_OnAggregate, next_timestamp, reports)) + + def _OnQueryReport(report): + if report is not None: + callback(report) + return + + # If the requested report doesn't already exist, it should be generated from metrics and previous reports. + HealthReport.QueryTimespan(client, group_key, oldest_report_timestamp, newest_report_timestamp, + _OnQueryPreviousReports) + + # Retrieve the report for the most recent time if it is not cached. + HealthReport.Query(client, group_key, newest_report_timestamp, None, _OnQueryReport, must_exist=False) diff --git a/backend/db/id_allocator.py b/backend/db/id_allocator.py new file mode 100644 index 0000000..a456192 --- /dev/null +++ b/backend/db/id_allocator.py @@ -0,0 +1,152 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""ID allocation from a monotonically increasing sequence. + +An instance of IdAllocator is created by specifying the id-allocation key. +This is typically a table name (e.g. 'users'), though can be +any arbitrary key. Each instance maintains a block of sequential IDs. The +size of the block can be controlled at instantiation but defaults to +IdAllocator._DEFAULT_ALLOCATION. + +Allocation of IDs starts at _START_ID (default is 1). + + IdAllocator: keeps track of a block of IDs, mapped from id_allocator table +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import struct +import zlib + +from collections import deque +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject + + +@DBObject.map_table_attributes +class IdAllocator(DBHashObject): + """Viewfinder ID allocator.""" + __slots__ = ['_allocation', '_next_id_key', '_cur_id', '_last_id', + '_allocation_pending', '_waiters'] + + _START_ID = 1 + _DEFAULT_ALLOCATION = 7 + + _table = DBObject._schema.GetTable(vf_schema.ID_ALLOCATOR) + _instances = list() + + def __init__(self, id_type=None, allocation=None): + """Allocates a block of 'allocation' IDs from the 'id_allocator' + table. + + Specify allocation as a prime number to make it less likely that + two allocating servers are handing out numbers with synchronized + mod offsets. This shouldn't in practice be an issue as the hash + prefix we compute is constructed via a crc32--synchronized mod + offsets shouldn't be a problem here. + """ + super(IdAllocator, self).__init__() + self.id_type = id_type + self._allocation = allocation or IdAllocator._DEFAULT_ALLOCATION + self._next_id_key = self._table.GetColumn('next_id').key + self._cur_id = IdAllocator._START_ID + self._last_id = IdAllocator._START_ID + self._allocation_pending = False + self._waiters = deque() + # Add to the global instance list so we can reset from unittest setup. + IdAllocator._instances.append(self) + + def Reset(self): + assert len(self._waiters) == 0, self._waiters + assert not self._allocation_pending + self._cur_id = IdAllocator._START_ID + self._last_id = IdAllocator._START_ID + self._allocation_pending = False + + def NextId(self, client, callback): + """Executes callback with the value of _cur_id++. If _cur_id is None or + _cur_id == _last_id, allocates a new block from the 'id_allocator' table. + """ + if self._allocation_pending: + self._waiters.append(partial(self._AllocateId, callback)) + elif self._cur_id == self._last_id: + self._waiters.append(partial(self._AllocateId, callback)) + self._AllocateIds(client) + else: + self._AllocateId(callback) + + def _AllocateId(self, callback, type=None, value=None, traceback=None): + """Invokes callback with a new id from the sequence; if type, value or + traceback are not None, raises an exception. + """ + if (type, value, traceback) != (None, None, None): + raise type, value, traceback + assert self._cur_id < self._last_id + new_id = self._cur_id + self._cur_id += 1 + callback(new_id) + + def _ProcessWaiters(self): + """Iterates over list of waiters, returning new ids from the + allocation stream. Returns true if all waiters were processed; + false otherwise. + """ + while len(self._waiters) and self._cur_id < self._last_id: + self._waiters.popleft()() + return len(self._waiters) == 0 + + def _AllocateIds(self, client): + """Allocates the next batch of IDs. On success, processes all + pending waiters. If there are more waiters than ids, re-allocates. + Otherwise, resets _allocation_pending. + """ + assert self._cur_id == self._last_id, (self._cur_id, self._last_id) + + def _OnAllocate(result): + self._last_id = result.return_values[self._next_id_key] + if self._last_id <= IdAllocator._START_ID: + self._cur_id = self._last_id + return self._AllocateIds(client) + self._cur_id = max(IdAllocator._START_ID, self._last_id - self._allocation) + assert self._cur_id < self._last_id, 'cur id %d >= allocated last id %d' % \ + (self._cur_id, self._last_id) + logging.debug("allocated %d %s IDs (%d-%d)" % + (self._allocation, self.id_type, self._cur_id, self._last_id)) + if not self._ProcessWaiters(): + self._AllocateIds(client) + else: + self._allocation_pending = False + + def _OnError(type, value, traceback): + logging.error('failed to allocate new id; returning waiters...', exc_info=(type, value, traceback)) + while len(self._waiters): + self._waiters.popleft()(type, value, traceback) + + self._allocation_pending = True + with util.MonoBarrier(_OnAllocate, on_exception=_OnError) as b: + client.UpdateItem(table=self._table.name, key=self.GetKey(), + attributes={self._next_id_key: + db_client.UpdateAttr(value=self._allocation, action='ADD')}, + return_values='UPDATED_NEW', callback=b.Callback()) + + @staticmethod + def ComputeHashPrefix(id, num_bytes=1): + """Returns a hash prefix from 64-bit id with the specified number + of bytes. The hash prefix for an ID is typically used to achieve + uniform distribution of keys across shards. The high bytes of the + crc32 checksum are used first. + """ + assert num_bytes - 1 in xrange(4), num_bytes + return zlib.crc32(struct.pack('>Q', id)) & [0xff, 0xffff, 0xffffff, 0xffffffff][num_bytes - 1] + + @classmethod + def ResetState(cls): + """Resets the internal state of all ID allocators; for testing.""" + for id_alloc in IdAllocator._instances: + id_alloc.Reset() + diff --git a/backend/db/identity.py b/backend/db/identity.py new file mode 100644 index 0000000..26f1703 --- /dev/null +++ b/backend/db/identity.py @@ -0,0 +1,453 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder identity. + +Identities are provided by OAuth authorities such as Google, Facebook +or Twitter. A viewfinder user account may have multiple identities for +a user. However, each identity may be associated with only one viewfinder +account. + +TODO(spencer): notice contacts no longer being fetched and delete them. + The hard part is figuring out a good way to communicate + this info to the client. The deletion just requires keeping + track of which contacts in the full contacts dict are no + longer being fetched. Not a huge priority as contacts are + not often deleted (though Facebook friends may be). + + Identity: viewfinder identity. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import calendar +import iso8601 +import json +import logging +import math +import phonenumbers +import time +import urllib + +from Crypto.Random import random +from functools import partial +from itertools import izip +from operator import itemgetter +from tornado import gen, httpclient, web +from viewfinder.backend.base import base64hex, constants, secrets, util +from viewfinder.backend.base.exceptions import ExpiredError, InvalidRequestError +from viewfinder.backend.base.exceptions import PermissionError, TooManyGuessesError +from viewfinder.backend.base.exceptions import TooManyRetriesError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.guess import Guess +from viewfinder.backend.db.hash_base import DBHashObject +from viewfinder.backend.db.contact import Contact +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.short_url import ShortURL +from viewfinder.backend.op.notification_manager import NotificationManager +from viewfinder.backend.resources.message.error_messages import BAD_IDENTITY, INCORRECT_ACCESS_CODE +from viewfinder.backend.www import www_util + + +EXPIRED_EMAIL_LINK_ERROR = 'The link in your email has expired or already been used. ' + \ + 'Please retry account sign up or log on.' + +EXPIRED_ACCESS_CODE_ERROR = 'The access code has expired or already been used. ' + \ + 'Please retry account sign up or log on.' + +TOO_MANY_GUESSES_ERROR = 'Your account has been locked for 24 hours, due to repeated unsuccessful attempts ' + \ + 'to log on. If this was not you, please e-mail support@emailscrubbed.com.' + + +@DBObject.map_table_attributes +class Identity(DBHashObject): + """Viewfinder identity.""" + + _IDENTITY_SCHEMES = ('FacebookGraph', 'Email', 'Phone', 'Local') + _GOOGLE_REFRESH_URL = 'https://accounts.google.com/o/oauth2/token' + + _TIME_TO_INVITIATION_EXPIRATION = constants.SECONDS_PER_DAY * 30 + """Expire prospective user links after 30 days.""" + + _MAX_GUESSES = 10 + """Maximum number of access token guesses before lock-out.""" + + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.IDENTITY) + + def __init__(self, key=None, user_id=None): + """Creates a new identity with the specified key.""" + super(Identity, self).__init__() + self.key = key + self.user_id = user_id + + @classmethod + def ShouldScrubColumn(cls, name): + return name in ('access_code', 'access_token', 'refresh_token') + + @classmethod + def ValidateKey(cls, identity_key): + """Validates that the identity key has a valid format and is canonicalized.""" + if Identity.Canonicalize(identity_key) != identity_key: + raise InvalidRequestError('Identity %s is not in canonical form.' % identity_key) + + @classmethod + def Canonicalize(cls, identity_key): + """Returns the canonical form of the given identity key.""" + for prefix in ['Email:', 'Phone:', 'FacebookGraph:', 'Local:', 'VF:']: + if identity_key.startswith(prefix): + value = identity_key[len(prefix):] + if prefix == 'Email:': + canonical_value = Identity.CanonicalizeEmail(value) + if value is not canonical_value: + identity_key = prefix + canonical_value + elif prefix == 'Phone:': + canonical_value = Identity.CanonicalizePhone(value) + if value is not canonical_value: + identity_key = prefix + canonical_value + + # Valid prefix, but no canonicalization necessary. + return identity_key + + raise InvalidRequestError('Scheme for identity %s unknown.' % identity_key) + + @classmethod + def CanonicalizeEmail(cls, email): + """Given an arbitrary string, validates that it is in legal email format. Normalizes + the email by converting it to lower case and returns it. + + TODO(Andy): Add validation that email at least contains the '@' symbol. + + Consistent with the iOS client's ContactManager::CanonicalizeEmail() function. + """ + return email if email.islower() else email.lower() + + @classmethod + def CanonicalizePhone(cls, phone): + """Given an arbitrary string, validates that it is in the expected E.164 phone number + format. Since E.164 phone numbers are already canonical, there is no additional + normalization step to take. Returns the valid, canonical phone number in E.164 format. + """ + if not phone: + raise InvalidRequestError('Phone number cannot be empty.') + + if phone[0] != '+': + raise InvalidRequestError('Phone number "%s" is not in E.164 format.' % phone) + + try: + phone_num = phonenumbers.parse(phone) + except phonenumbers.phonenumberutil.NumberParseException: + raise InvalidRequestError('"%s" is not a phone number.' % phone) + + if not phonenumbers.is_possible_number(phone_num): + raise InvalidRequestError('"%s" is not a possible phone number.' % phone) + + return phone + + @classmethod + def CanCanonicalizePhone(cls, phone): + """Given an arbitrary string, checks that it is in the expected E.164 phone number + format. + Returns: True if phone number can be successfully canonicalized. + """ + try: + Identity.CanonicalizePhone(phone) + except InvalidRequestError: + return False + return True + + @classmethod + @gen.coroutine + def CreateProspective(cls, client, identity_key, user_id, timestamp): + """Creates identity for a new prospective user. This typically happens when photos are + shared with a contact that is not yet a Viewfinder user. + """ + # Make sure that identity is not being bound to a different user. + identity = yield gen.Task(Identity.Query, client, identity_key, None, must_exist=False) + if identity is None: + identity = Identity.CreateFromKeywords(key=identity_key) + else: + assert identity.user_id is None or identity.user_id == user_id, \ + 'the identity is already in use: %s' % identity + + identity.user_id = user_id + + # Prospective user linking always done by authority of Viewfinder. + identity.authority = 'Viewfinder' + + yield gen.Task(identity.Update, client) + + # Update all contacts that refer to this identity. + yield identity._RewriteContacts(client, timestamp) + + raise gen.Return(identity) + + @classmethod + @gen.coroutine + def CreateInvitationURL(cls, client, user_id, identity_key, viewpoint_id, default_url): + """Creates and returns a prospective user invitation ShortURL object. The URL is handled + by an instance of AuthProspectiveHandler, which is "listening" at "/pr/...". The ShortURL + group is partitioned by user id so that incorrect guesses only affect a single user. + """ + identity_type, identity_value = Identity.SplitKey(identity_key) + now = util.GetCurrentTimestamp() + expires = now + Identity._TIME_TO_INVITIATION_EXPIRATION + encoded_user_id = base64hex.B64HexEncode(util.EncodeVarLengthNumber(user_id), padding=False) + short_url = yield ShortURL.Create(client, + group_id='pr/%s' % encoded_user_id, + timestamp=now, + expires=expires, + identity_key=identity_key, + viewpoint_id=viewpoint_id, + default_url=default_url, + is_sms=identity_type == 'Phone') + + raise gen.Return(short_url) + + @gen.coroutine + def CreateAccessTokenURL(self, client, group_id, use_short_token, **kwargs): + """Creates a verification access token. + + The token is associated with a ShortURL that will be sent to the identity email address + or phone number. Following the URL will reveal the access token. The user that presents + the correct token to Identity.VerifyAccessToken is assumed to be in control of that email + address or SMS number. + + Returns the ShortURL that was generated. + """ + identity_type, value = Identity.SplitKey(self.key) + num_digits, good_for = Identity.GetAccessTokenSettings(identity_type, use_short_token) + + now = util.GetCurrentTimestamp() + access_token = None + if self.authority == 'Viewfinder' and now < self.expires and \ + self.access_token is not None and len(self.access_token) == num_digits: + # Re-use the access token. + access_token = self.access_token + + if access_token is None: + # Generate new token, which is a random decimal number of 4 or 9 decimal digits. + format = '%0' + str(num_digits) + 'd' + access_token = format % random.randint(0, 10 ** num_digits - 1) + + # Create a ShortURL that contains the access token, along with caller-supplied parameters. + expires = now + good_for + short_url = yield ShortURL.Create(client, + group_id, + timestamp=now, + expires=expires, + access_token=access_token, + **kwargs) + + # Update the identity to record the access token and short url information. + self.authority = 'Viewfinder' + self.access_token = access_token + self.expires = expires + self.json_attrs = {'group_id': short_url.group_id, 'random_key': short_url.random_key} + yield gen.Task(self.Update, client) + + # Check whether user account is locked due to too many guesses. + guess_id = self._ConstructAccessTokenGuessId(identity_type, self.user_id) + if not (yield Guess.CheckGuessLimit(client, guess_id, Identity._MAX_GUESSES)): + raise TooManyGuessesError(TOO_MANY_GUESSES_ERROR) + + raise gen.Return(short_url) + + @gen.coroutine + def VerifyAccessToken(self, client, access_token): + """Verifies the correctness of the given access token, that was previously generated in + response to a CreateAccessTokenURL call. Verification will fail if any of these conditions + is false. + + 1. The access token is expired. + 2. Too many incorrect attempts to guess the token have been made in the past. + 3. The access token does not match. + """ + identity_type, identity_value = Identity.SplitKey(self.key) + now = time.time() + + if identity_type == 'Email': + error = ExpiredError(EXPIRED_EMAIL_LINK_ERROR) + else: + error = ExpiredError(EXPIRED_ACCESS_CODE_ERROR) + + if self.authority != 'Viewfinder': + # The most likely case here is that the user clicked an old link in their inbox. In the interim since + # receiving the link, they may have logged in with Google, which would update the authority to Google. + # In this case, the ExpiredError is an appropriate error message since the link is expired. + logging.warning('the authority is not "Viewfinder" for identity "%s"', self.key) + raise error + + if now >= self.expires: + # Either the access token has expired, or has already been used up. + logging.warning('the access token has expired for identity "%s"', self.key) + raise error + + # Fail if too many incorrect guesses have been made. + guess_id = self._ConstructAccessTokenGuessId(identity_type, self.user_id) + if not (yield Guess.CheckGuessLimit(client, guess_id, Identity._MAX_GUESSES)): + logging.warning('too many access token guesses have been made for identity "%s"', self.key) + raise TooManyGuessesError(TOO_MANY_GUESSES_ERROR) + + # Increment incorrect guess account and raise permission error if the access code did not match. + if not web._time_independent_equals(self.access_token, access_token): + logging.warning('the access token "%s" does not match for identity "%s"', access_token, self.key) + yield Guess.ReportIncorrectGuess(client, guess_id) + raise PermissionError(INCORRECT_ACCESS_CODE, identity_value=Identity.GetDescription(self.key)) + + @classmethod + @gen.coroutine + def VerifyConfirmedIdentity(cls, client, identity_key, access_token): + """Verifies that the specified access token matches the one stored in the identity. If + this is the case, then the caller has confirmed control of the identity. Returns the + identity DB object if so, else raises a permission exception. + """ + # Validate the identity and access token. + Identity.ValidateKey(identity_key) + identity = yield gen.Task(Identity.Query, client, identity_key, None, must_exist=False) + if identity is None: + raise InvalidRequestError(BAD_IDENTITY, identity_key=identity_key) + + yield identity.VerifyAccessToken(client, access_token) + + # Expire the access token now that it has been used. + identity.expires = 0 + + # Reset auth throttling limit since access token has been successfully redeemed. + identity.auth_throttle = None + + identity.Update(client) + + raise gen.Return(identity) + + @classmethod + @gen.coroutine + def UnlinkIdentity(cls, client, user_id, key, timestamp): + """Unlinks the specified identity from the account identified by 'user_id'. Queries all + contacts which reference the identity and update their timestamps so that they will be + picked up by query_contacts. + """ + identity = yield gen.Task(Identity.Query, client, key, None) + assert identity.user_id is None or identity.user_id == user_id, identity + yield identity._RewriteContacts(client, timestamp) + yield gen.Task(identity.Delete, client) + + @classmethod + def GetDescription(cls, identity_key): + """Returns a description of the specified identity key suitable for UI display.""" + identity_type, value = Identity.SplitKey(identity_key) + if identity_type == 'Email': + return value + elif identity_type == 'FacebookGraph': + return 'your Facebook account' + elif identity_type == 'Phone': + phone = phonenumbers.parse(value) + return phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.INTERNATIONAL) + elif identity_type == 'Local': + return 'local identity' + raise InvalidRequestError('Scheme for identity %s unknown.' % identity_key) + + @classmethod + def SplitKey(cls, identity_key): + """Splits the given identity key of the form : and returns the (type, value) + as a tuple. + """ + return identity_key.split(':', 1) + + @classmethod + def GetAccessTokenSettings(cls, identity_type, use_short_token): + """Returns settings that control how the access token for various identity types behaves. + The settings are returned as a tuple: + + (digit_count, good_for) + + digit_count: number of decimal digits in the access code + good_for: time span (in seconds) during which the token is accepted + """ + if identity_type == 'Phone' or use_short_token: + # 4-digit token, expire token after an hour. + return (4, constants.SECONDS_PER_HOUR) + elif identity_type == 'Email': + # 9-digit token, expire token after a day. + return (9, constants.SECONDS_PER_DAY) + + assert False, 'unsupported identity type "%s"' % identity_type + + def RefreshGoogleAccessToken(self, client, callback): + """Refreshes an expired google access token using the refresh token. + """ + def _OnRefresh(response): + try: + response_dict = www_util.ParseJSONResponse(response) + except web.HTTPError as e: + if e.status_code == 400: + logging.error('%s: failed to refresh access token; clearing refresh token' % e) + with util.ExceptionBarrier(util.LogExceptionCallback): + self.refresh_token = None + self.Update(client, util.NoCallback) + raise + + self.access_token = response_dict['access_token'] + self.expires = time.time() + response_dict['expires_in'] + callback() + + body = urllib.urlencode({'refresh_token': self.refresh_token, + 'client_id': secrets.GetSecret('google_client_id'), + 'client_secret': secrets.GetSecret('google_client_secret'), + 'grant_type': 'refresh_token'}) + http_client = httpclient.AsyncHTTPClient() + http_client.fetch(Identity._GOOGLE_REFRESH_URL, method='POST', callback=_OnRefresh, body=body) + + @classmethod + def _ConstructAccessTokenGuessId(cls, identity_type, user_id): + """Constructs an access token guess id value, used to limit the number of incorrect guesses + that can be made for a particular identity type + user. + """ + if identity_type == 'Email': + return Guess.ConstructGuessId('em', user_id) + + assert identity_type == 'Phone', identity_type + return Guess.ConstructGuessId('ph', user_id) + + @gen.coroutine + def _RewriteContacts(self, client, timestamp): + """Rewrites all contacts which refer to this identity. All timestamps are updated so that + query_contacts will pick them up. + """ + @gen.coroutine + def _RewriteOneContact(co): + """Update the given contact's timestamp.""" + new_co = Contact.CreateFromKeywords(co.user_id, + co.identities_properties, + timestamp, + co.contact_source, + name=co.name, + given_name=co.given_name, + family_name=co.family_name, + rank=co.rank) + + # Only rewrite if timestamp is different. + if co.sort_key != new_co.sort_key: + yield gen.Task(new_co.Update, client) + yield gen.Task(co.Delete, client) + + query_expr = ('contact.identities={id}', {'id': self.key}) + contacts = yield gen.Task(Contact.IndexQuery, client, query_expr, None) + + # Update each contact which points to this identity. + yield [_RewriteOneContact(co) for co in contacts] + + @classmethod + @gen.coroutine + def UnlinkIdentityOperation(cls, client, user_id, identity): + """Unlinks the specified identity from any associated viewfinder user.""" + # All contacts created during UnlinkIdentity are based on the current operation's timestamp. + timestamp = Operation.GetCurrent().timestamp + + yield Identity.UnlinkIdentity(client, user_id, identity, timestamp) + + # Notify clients of any contacts that have been updated. + yield NotificationManager.NotifyUnlinkIdentity(client, user_id, identity, timestamp) diff --git a/backend/db/indexers.py b/backend/db/indexers.py new file mode 100644 index 0000000..321a17f --- /dev/null +++ b/backend/db/indexers.py @@ -0,0 +1,388 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Indexers. + +Indexers transform database column (aka attribute) values into index +terms for a reverse index. + + Indexer: creates secondary index(es) for a column [abstract] + SecondaryIndexer: simplest indexer for implementing secondary indexes + LocationIndexer: indexes location across all S2 cell resolutions + BreadcrumbIndexer: indexes location at a specific (50m-radius) S2 resolution + LocationIndexer: indexes location by emitting S2 patches + PlacemarkIndexer: indexes hierarchical place names + FullTextIndexer: separate col value via white-space for full-text search + EmailTokenizer: tokenizes email addresses +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import s2 +import re +import struct + +from viewfinder.backend.base import base64hex +from viewfinder.backend.base.util import ConvertToString +from viewfinder.backend.db import stopwords + +try: + # We have two double metaphone implementations available: + # The one in "fuzzy" is faster, but doesn't work on pypy. + import fuzzy + _D_METAPHONE = fuzzy.DMetaphone() +except ImportError: + import metaphone + _D_METAPHONE = metaphone.doublemetaphone + +class Indexer(object): + """An indexer creates arbitrary secondary indexes for a column by + transforming the column value into a set of index terms. Each index + term will be stored as a link back to the object containing the + column. The set of index terms are actually the keys to a python + dict, with value being an opaque datum to be retrieved in addition + to the primary key of the object (more on the utility of this below). + + The simplest example of an Indexer would return the exact value of + the column. This is equivalent to creating a secondary key on the + column in a relational database. The object can now be queried by + this column's value in addition to the primary key value. + + Full text search can be implemented by parsing column values by + whitespace, and emitting the resulting words as index terms. In this + case, the datum accompanying the index terms would be a list of word + positions, for implementing phrase searches. + + Another illustrative example would be an indexer for location, as + specified by a (longitude, latitude, accuracy) tuple. This might + yield a set of S2 geometry patches, each locating the image in a + successively more exact region of the earth from, for example, + continent to city block. + + Only "field-specific" index terms are allowed. This means that every + index term emitted while indexing a column is prefixed with the + table key and the column key. This prevents term collisions between + tables--and within a table--between columns, which would otherwise + confuse or break phrase searches, and would cause headaches when + updating columns (you'd either need to ref-count or to do wholesale + updates/deletes of objects instead of allowing incremental updates. + + Columns which hold a set of values (e.g. a set of user email addresses + used as login identities), may only use the SecondaryIndexer, which + very simply emits the column value as the only index term. Since the + column is a set, the column value yields a unique index term--which + means it works properly with incremental set additions and deletions + so we can have arbitrarily large sets. If you do need to whitespace + tokenize an email address (to continue the example), then create a + field(s) for primary email address, secondary email address, etc., + which can set whatever tokenizer they like. + """ + + class Option: + """Class enum for whether this tokenizer should include various + expansions. The allowed values are one of: + + 'NO': do not include terms for this option + 'YES': include terms both for this option and without + 'ONLY': only include terms for this option + """ + NO, YES, ONLY = range(3) + + def Index(self, col, value): + """Parses the provided value into a dict of {tokenized term: + freighted data}. By default, emits the column value as the only + index term. Subclasses override for specific behavior. + """ + return dict([(t, None) for t in self._ExpandTerm(col, value)]) + + def GetQueryString(self, col, value): + """Returns a query string suitable for the query parser to match + the specified value. In most cases, this is simply the term + itself, prefixed with the key + ':'. However, for full-text + search, this would generate a succession of 'and's for phrase + searches and potentially 'or's in cases where a term has homonyms, + as in metaphone expansions. + """ + exp_terms = self._ExpandTerm(col, value) + return exp_terms[0] if len(exp_terms) == 1 \ + else '(' + ' | '.join(exp_terms) + ')' + + def UnpackFreight(self, col, posting): + """Returns a value representing the unpacked contents of data that + were freighted with the posting of this term. This is tokenizer- + dependent. For example, the FullTextIndexer freights a list + of word positions. + """ + return None + + def _InterpretOption(self, option, term, optional_term): + """Returns either the first, second or both terms from the list + depending on the value of option. + """ + if option == Indexer.Option.NO: + return [term] + elif option == Indexer.Option.ONLY: + return [optional_term] + elif option == Indexer.Option.YES: + return [term, optional_term] + raise TypeError() + + def _ExpandTerm(self, col, term): + """Expands each term in 'terms'. In the base class, this merely + prepends the table key + ':' + column key + ':' to each term. + """ + prefix = col.key + ':' + if col.table: + prefix = col.table.key + ':' + prefix + return [prefix + ConvertToString(term)] + + +class SecondaryIndexer(Indexer): + """An indexer class which simply emits the column value.""" + def GetQueryString(self, col, value): + """Returns a quoted string to match the column value exactly. + """ + exp_terms = self._ExpandTerm(col, value) + assert len(exp_terms) == 1 + return '"%s"' % exp_terms[0] + + +class TimestampIndexer(Indexer): + """An indexer class which emits tokens to support queries over + time intervals. + TODO(spencer): fix this; currently only works with exact time + """ + def Index(self, col, value): + """Parses the provided value into a dict of {tokenized term: + freighted data}. By default, emits the column value as the only + index term. Subclasses override for specific behavior. + """ + return dict([(t, None) for t in self._ExpandTerm(col, int(value))]) + + def GetQueryString(self, col, value): + """ + """ + exp_terms = self._ExpandTerm(col, int(value)) + assert len(exp_terms) == 1 + return '"%s"' % exp_terms[0] + + +class BreadcrumbIndexer(Indexer): + """Indexer for user breadcrumbs. On indexing, each breadcrumb + generates a sequence of S2 geometry cells at the specified + S2_CELL_LEVEL cell level to cover a radius of + RADIUS. On query, only a single patch is generated at + S2_CELL_LEVEL to minimize the search read requirements. + """ + RADIUS = 51 + S2_CELL_LEVEL = s2.GetClosestLevel(RADIUS) + + def Index(self, col, value): + """Generates implicated S2 patches at S2_CELL_LEVEL which cover an + S2 cap centered at lat/lon with radius RADIUS. + """ + lat, lon, acc = value + cells = [c for c in s2.SearchCells( + lat, lon, BreadcrumbIndexer.RADIUS, + BreadcrumbIndexer.S2_CELL_LEVEL, BreadcrumbIndexer.S2_CELL_LEVEL)] + assert len(cells) <= 10, len(cells) + return dict([(t, None) for c in cells for t in self._ExpandTerm(col, c)]) + + def GetQueryString(self, col, value): + """The provided value is a latitude, longitude, accuracy + tuple. Returns a search query for the indicated S2_CELL_LEVEL cell. + """ + lat, lon, acc = [float(x) for x in value.split(',')] + cells = s2.IndexCells(lat, lon, BreadcrumbIndexer.S2_CELL_LEVEL, + BreadcrumbIndexer.S2_CELL_LEVEL) + assert len(cells) == 1, [repr(c) for c in cells] + exp_terms = self._ExpandTerm(col, cells[0]) + return exp_terms[0] if len(exp_terms) == 1 \ + else '(' + ' | '.join(exp_terms) + ')' + + +class LocationIndexer(Indexer): + """An indexer class which emits S2 patch values at various + resolutions corresponding to a latitude/longitude/accuracy tuple. + + The resolution goes from 10m to 1000km, which is roughly s2 patch + levels 3 to 25. + """ + _S2_MIN = 3 + _S2_MAX = 25 + + def Index(self, col, value): + """Generates implicated S2 patches from levels (_S2_MIN, _S2_MAX). + """ + lat, lon, acc = value + cells = [c for c in s2.IndexCells( + lat, lon, LocationIndexer._S2_MIN, LocationIndexer._S2_MAX)] + cells.reverse() + return dict([(t, None) for c in cells for t in self._ExpandTerm(col, c)]) + + def GetQueryString(self, col, value): + """The provided value is a triplet of latitude, longitude and a + radius in meters. Returns an 'or'd set of S2 geometry patch terms + that cover the region. + """ + lat, lon, rad = [float(x) for x in value.split(',')] + cells = [c for c in s2.SearchCells(lat, lon, rad, LocationIndexer._S2_MIN, + LocationIndexer._S2_MAX)] + exp_terms = [t for c in cells for t in self._ExpandTerm(col, c)] + return exp_terms[0] if len(exp_terms) == 1 \ + else '(' + ' | '.join(exp_terms) + ')' + + +class PhraseSearchIndexer(Indexer): + """A base class for indexing values where relative position of + tokens is important and the indexer must support phrase searches, + such as the full-text indexer and the placemark indexer. + + Sub-classes must implement _Tokenize(). + """ + def Index(self, col, value): + """Returns words as contiguous alpha numeric strings (and + apostrophes) which are of length > 1 and are also not in the stop + words list. Each term is freighted with a list of term positions + (formatted as a packed binary string). + """ + terms = {} + expansions = {} # map from term to expanded set of terms + tokens = self._Tokenize(value) + for pos, term in zip(xrange(len(tokens)), tokens): + if term == '_': + continue + if term not in expansions: + expansions[term] = self._ExpandTerm(col, term) + for exp_term in expansions[term]: + if not terms.has_key(exp_term): + terms[exp_term] = '' + if pos < 1<<16: + terms[exp_term] += struct.pack('>H', pos) + + # Base64Hex Encode positions. + for k,v in terms.items(): + if v: + terms[k] = base64hex.B64HexEncode(v, padding=False) + + return terms + + def GetQueryString(self, col, value): + """Returns a query string suitable for the query parser to match + the specified value. If the value tokenizes to multiple terms, + generates a conjunction of '+' operators which is like '&", but + with a positional requirement (this implements phrase + search). Each term is then expanded into a conjunction of 'or' + operators. + """ + def _GetExpansionString(term): + if term == '_': + return term + exp_terms = self._ExpandTerm(col, term) + if len(exp_terms) == 1: + return exp_terms.pop() + else: + return '(' + ' | '.join(exp_terms) + ')' + + tokens = self._Tokenize(value) + if len(tokens) == 1: + return _GetExpansionString(tokens[0]) + else: + return '(' + ' + '.join([_GetExpansionString(token) for token in tokens]) + ')' + + def UnpackFreight(self, col, posting): + # TODO(spencer): the rstrip() below is necessary as data in the + # index has already been encoded with a bug in the base64 padding + # We need to rebuild the index before reverting this. + posting = base64hex.B64HexDecode(posting.rstrip('='), padding=False) + assert not (len(posting) % 2), repr(posting) + return [struct.unpack('>H', posting[i:i+2])[0] for i in xrange(0, len(posting), 2)] + + +class PlacemarkIndexer(PhraseSearchIndexer): + """An indexer class which emits index terms for each hierarchical + name in a placemark structure. + + Phrase searching is supported by reversing the placemark names from + least to most specific (so "Paris, France" and "New York, NY" work + properly). + + TODO(spencer): provide an additional mechanism for searching + specifically for country=X, etc. + """ + _SPLIT_CHARS = re.compile("[^a-z0-9 ]") + + def Index(self, col, value): + places = value._asdict().values() + places.reverse() + return super(PlacemarkIndexer, self).Index(col, ' '.join(places)) + + def _Tokenize(self, value): + """Strips all punctuation characters and tokenizes by whitespace.""" + return PlacemarkIndexer._SPLIT_CHARS.sub('', value.lower()).split() + + +class FullTextIndexer(PhraseSearchIndexer): + """An Indexer class meant for creating index terms for full-text + search. Provides an optional facility for normalizing english words + to a phonetic system to easily correct for mispellings. + + Uses non-alpha-numeric characters to split the value into words, + filters out English stop words, and returns a sequence of (term, + struct-packed position) pairs. The struct-packed position list uses + two bytes for each position. It only applies to the first 65536 + words. + + Shout out to: http://dr-josiah.blogspot.com/2010/07/ + building-search-engine-using-redis-and.html + """ + _SPLIT_CHARS = re.compile("[^a-z0-9' ]") + + def __init__(self, metaphone=Indexer.Option.NO): + """- metaphone: generate metaphone query terms. Metaphone is an + expansive phonetic representation of english language words. + """ + super(FullTextIndexer, self).__init__() + self._metaphone = metaphone + + def _Tokenize(self, value): + """Splits 'value' into a sequence of (position, token) tuples + according to whitespace. Stop words are represented by the '_' character. + """ + tokens = FullTextIndexer._SPLIT_CHARS.sub(' ', value.lower()).split() + tokens = [token.strip("'") for token in tokens] + for i in xrange(len(tokens)): + if tokens[i] in stopwords.STOP_WORDS or len(tokens[i]) == 1: + tokens[i] = '_' + return tokens + + def _ExpandTerm(self, col, term): + """Expand term according to metaphone and then for each, expand + using Indexer._ExpandTerm. + """ + terms = set() + for meta_term in self.__ExpandMetaphone(term): + if meta_term: + terms = terms.union(Indexer._ExpandTerm(self, col, meta_term)) + return terms + + def __ExpandMetaphone(self, term): + """Expands term according to metaphone setting. Need to be careful + here about the metaphone algorithm returning no matches, as is the + case with numbers and sufficiently non-English words. In this + case, where there are no metaphone results, we just add the term. + """ + if self._metaphone == Indexer.Option.NO: + return set([term]) + else: + terms = set() + for dmeta_term in _D_METAPHONE(term): + if dmeta_term: + terms = terms.union(self._InterpretOption( + self._metaphone, term, dmeta_term)) + if not terms: + terms.add(term) + return terms + + +class EmailIndexer(FullTextIndexer): + pass diff --git a/backend/db/job.py b/backend/db/job.py new file mode 100644 index 0000000..3c3435a --- /dev/null +++ b/backend/db/job.py @@ -0,0 +1,220 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Job utility functions. + +A Job is a process independent of the backend but using dynamodb to coordinate runs. +The class provides two independent (and both optional) features: +- locking: grab a job-specific lock for the duration of the work. prevents other instances of the same job from running. + methods: AcquireLock(), ReleaseLock() +- run status: fetch previous run statuses and write run status at the end (with optional additional stats). + method: Start(), FindPreviousRuns(), RegisterRun() + +Jobs should be used as follows: + +job = Job(db_client, 'dbchk') +if yield gen.Task(job.AcquireLock): + # Acquired lock, we must release it eventually. + # Record start time: + job.Start() + + # Find last successful run within the last 7 days. + runs = yield gen.Task(job.FindPreviousRuns, start_timestamp=time.time() - 7*24*3600, + status=Job.STATUS_SUCCESS, limit=1) + if len(runs) > 0: + yield gen.Task(job.ReleaseLock) + return + status = Job.STATUS_SUCCESS + try: + # do work + except: + status = Job.STATUS_FAILURE + + # Write summary with extra stats. + stats = DotDict() + stats['analyzed_bytes': 1024] + yield gen.Task(job.RegisterRun(status, stats=stats): + + # Release lock. + yield gen.Task(job.ReleaseLock) + +By default, abandoned locks can be acquired. Only jobs that cannot recover from a failed run and require manual +intervention should set detect_abandonment=False. + +TODO(marc): add helpers to read/write job status entries in the Metric table. This is how we're going +to figure out what the last successful run was and determine whether to run again. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import json +import logging +import os +import time + +from tornado import gen +from viewfinder.backend.base import constants, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, metric +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType + +class Job(object): + + # Status codes stored in the metric payload.status + STATUS_SUCCESS = 'success' + STATUS_FAILURE = 'failure' + + def __init__(self, client, name): + """Initialize a new Job object.""" + self._client = client + self._name = name + self._lock = None + self._start_time = None + + + def __del__(self): + """There's no point in attempting to release the lock in the destructor, it's much too unreliable.""" + if self._lock is not None: + logging.error('Job:%s is being cleaned, but lock was not released: %r' % (self._name, self._lock)) + + + def Start(self): + """Start the start time to now.""" + self._start_time = int(time.time()) + + + def HasLock(self): + """Return true if a lock is held.""" + return self._lock is not None + + @gen.engine + def AcquireLock(self, callback, resource_data=None, detect_abandonment=True): + """Attempt to acquire the lock "job:name". Returns True if acquired. + If resource_data is None, a string is built consisting of the local user name, + machine hostname and timestamp. + If detect_abandonment is True, we allow acquisition of expired locks and specify + an expiration on our lock. If False, acquiring an abandoned lock is considered an acquire failure. + If lock acquisition succeeds, the client must call Release() when finished to release the lock. + """ + assert self._lock is None, 'Job.AcquireLock called with existing lock held %r' % self._lock + + data = '%s@%s:%d' % (util.GetLocalUser(), os.uname()[1], time.time()) if resource_data is None else resource_data + + result = yield gen.Task(Lock.TryAcquire, self._client, LockResourceType.Job, self._name, + resource_data=data, detect_abandonment=detect_abandonment) + lock, status = result.args + + if status == Lock.FAILED_TO_ACQUIRE_LOCK: + callback(False) + elif status == Lock.ACQUIRED_ABANDONED_LOCK and not detect_abandonment: + logging.warning('Acquired abandoned lock, but specified locking without expiration; abandoning %r' % lock) + yield gen.Task(lock.Abandon, self._client) + callback(False) + else: + # ACQUIRED_LOCK or ACQUIRED_ABANDONED with abandonment detection enabled. + self._lock = lock + callback(True) + + + @gen.engine + def ReleaseLock(self, callback): + """Release _lock if not None.""" + if self._lock is not None: + yield gen.Task(self._lock.Release, self._client) + self._lock = None + callback() + + + @gen.engine + def FindPreviousRuns(self, callback, start_timestamp=None, status=None, limit=None): + """Look for previous runs of this job in the metrics table. Return all found runs regardless of status. + If start_timestamp is None, search for jobs started in the last week. + If status is specified, only return runs that finished with this status, otherwise return all runs. + If limit is not None, return only the latest 'limit' runs, otherwise return all runs. + Runs are sorted by timestamp. + """ + assert status in [None, Job.STATUS_SUCCESS, Job.STATUS_FAILURE], 'Unknown status: %s' % status + runs = [] + cluster = metric.JOBS_STATS_NAME + # TODO(marc): there is no guarantee that jobs will run daily (could be more or less). It shouldn't matter except + # when accessing the data using counters. + group_key = metric.Metric.EncodeGroupKey(cluster, metric.Metric.FindIntervalForCluster(cluster, 'daily')) + start_time = start_timestamp if start_timestamp is not None else time.time() - constants.SECONDS_PER_WEEK + + # Search for metrics from start_time to now. + existing_metrics = yield gen.Task(metric.Metric.QueryTimespan, self._client, group_key, start_time, None) + for m in existing_metrics: + if m.machine_id != self._name: + # Not for this job. + continue + + # Parse and validate payload. + payload = DotDict(json.loads(m.payload)) + assert 'start_time' in payload and 'status' in payload, 'Malformed payload: %r' % payload + assert payload['start_time'] == m.timestamp, 'Payload start_time does not match metric timestamp' + + if status is not None and payload['status'] != status: + continue + + runs.append(payload) + + # Sort by timestamp, although it should already should be. + runs.sort(key=lambda payload: payload['start_time']) + if limit is None: + callback(runs) + else: + callback(runs[-limit:]) + + + @gen.engine + def FindLastSuccess(self, callback, start_timestamp=None, with_payload_key=None, with_payload_value=None): + """Find and return the latest successful run. Search back to start_timestamp (a week ago if None). + If with_payload_key is not None, the key must be found in the payload (DotDict format). + If with_payload_value is not None, the value at that key must match. + Callback is run with the matching metric payload if found, else with None. + """ + payloads = yield gen.Task(self.FindPreviousRuns, start_timestamp=start_timestamp, status=Job.STATUS_SUCCESS) + for p in reversed(payloads): + assert p['status'] == Job.STATUS_SUCCESS + if with_payload_key is not None and with_payload_key not in p: + continue + if with_payload_value is not None: + assert with_payload_key is not None, 'with_payload_value specified, but with_payload_key is None' + if p[with_payload_key] != with_payload_value: + continue + callback(p) + return + + callback(None) + + + @gen.engine + def RegisterRun(self, status, callback, stats=None, failure_msg=None): + """Write the metric entry for this run. The start_time is set in Start(). end_time is now. + If stats is not none, the DotDict is added to the metrics payload with the prefix 'stats'. + If failure_msg is not None and status==STATUS_FAILURE, write the message in payload.failure_msg. + """ + assert status in [None, Job.STATUS_SUCCESS, Job.STATUS_FAILURE], 'Unknown status: %s' % status + assert self._start_time is not None, 'Writing job summary, but Start never called.' + end_time = int(time.time()) + payload = DotDict() + payload['start_time'] = self._start_time + payload['end_time'] = end_time + payload['status'] = status + if stats is not None: + assert isinstance(stats, DotDict), 'Stats is not a DotDict: %r' % stats + payload['stats'] = stats + if failure_msg is not None and status == Job.STATUS_FAILURE: + payload['failure_msg'] = failure_msg + + + cluster = metric.JOBS_STATS_NAME + group_key = metric.Metric.EncodeGroupKey(cluster, metric.Metric.FindIntervalForCluster(cluster, 'daily')) + new_metric = metric.Metric.Create(group_key, self._name, self._start_time, json.dumps(payload)) + yield gen.Task(new_metric.Update, self._client) + + # Clear start time, we should not be able to run RegisterRun multiple times for a single run. + self._start_time = None + + callback() diff --git a/backend/db/local_client.py b/backend/db/local_client.py new file mode 100644 index 0000000..6fbd70f --- /dev/null +++ b/backend/db/local_client.py @@ -0,0 +1,497 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Local emulation of DynamoDB client. + +Implements the DBClient interface identically (or as near as possible) +to the behavior expected from DynamoDB but using in-memory python data +structures for storage. +""" + +__author__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +from functools import partial +from bisect import bisect_left, bisect_right +import copy +import time + +from tornado.ioloop import IOLoop +from viewfinder.backend.base.exceptions import DBConditionalCheckFailedError +from db_client import DBClient, DBKey, ListTablesResult, CreateTableResult, DescribeTableResult, DeleteTableResult, GetResult, PutResult, DeleteResult, UpdateResult, QueryResult, ScanResult, BatchGetResult, TableSchema, UpdateAttr + +from viewfinder.backend.db import local_persist + +class LocalClient(DBClient): + """Local client for testing. + + - Datastore: dictionary of name => (HashTable, RangeTable) + - HashTable: dictionary of hash_key => Item + - Item: dictionary of attribute => value + - RangeTable: dictionary of hash_key => dictionary of range_key => Item + - Item: dictionary of attribute => value + """ + _MUTATING_RESULTS = [CreateTableResult, DeleteTableResult, PutResult, DeleteResult, UpdateResult] + + def __init__(self, schema, read_only=False): + self._schema = schema + self._read_only = read_only + self._tables = {} + self._table_schemas = {} + self._persist = local_persist.DBPersist(self._tables, self._table_schemas) + + def Shutdown(self): + """Shutdown persistence on process exit.""" + self._persist.Shutdown() + + def ListTables(self, callback): + table_names = [self._schema.TranslateNameInDb(name_in_db) + for name_in_db in self._tables.keys()] + result = ListTablesResult(tables=table_names) + return self._HandleCallback(callback, result) + + def CreateTable(self, table, hash_key_schema, range_key_schema, + read_units, write_units, callback): + assert not self._read_only, 'Received "CreateTable" request on read-only database' + + assert table not in self._tables, 'table %s already exists' % table + self._tables[table] = dict() + schema = TableSchema(create_time=time.time(), hash_key_schema=hash_key_schema, + range_key_schema=range_key_schema, read_units=read_units, + write_units=write_units, status='CREATING') + self._table_schemas[table] = self._NewSchemaStatus(schema, 'ACTIVE') + result = CreateTableResult(schema=schema) + return self._HandleCallback(callback, result) + + def DeleteTable(self, table, callback): + assert not self._read_only, 'Received "DeleteTable" request on read-only database' + + assert table in self._tables, 'table %s does not exist' % table + del self._tables[table] + del_schema = self._NewSchemaStatus(self._table_schemas[table], 'DELETING') + del self._table_schemas[table] + result = DeleteTableResult(schema=del_schema) + return self._HandleCallback(callback, result) + + def DescribeTable(self, table, callback): + assert table in self._tables, 'table %s does not exist' % table + result = DescribeTableResult(schema=self._table_schemas[table], + count=self._GetTableSize(table), size_bytes=None) + return self._HandleCallback(callback, result) + + def GetItem(self, table, key, callback, attributes, must_exist=True, + consistent_read=False): + self._CheckKey(table, key, True if must_exist else None, None) + if key.hash_key not in self._tables[table]: + return self._HandleCallback(callback, None) + item = self._tables[table][key.hash_key] + if key.range_key is not None: + if key.range_key not in item: + return self._HandleCallback(callback, None) + item = item[key.range_key] + result = GetResult(attributes=self._GetAttributes(item, attributes), read_units=1) + return self._HandleCallback(callback, result) + + def BatchGetItem(self, batch_dict, callback, must_exist=True): + assert len(batch_dict) == 1, 'BatchGetItem currently supports only a single table' + table_name, (keys, attributes, consistent_read) = next(batch_dict.iteritems()) + + result_items = [] + for key in keys: + result = self.GetItem(table_name, key, None, attributes, + must_exist=must_exist, consistent_read=consistent_read) + result_items.append(result.attributes if result is not None else None) + + result = {table_name: BatchGetResult(items=result_items, read_units=len(keys))} + return self._HandleCallback(callback, result) + + def PutItem(self, table, key, callback, attributes, expected=None, return_values=None): + assert not self._read_only, 'Received "PutItem" request on read-only database' + + self._CheckKey(table, key, None, expected) + item = self._GetItem(table, key) + # Make sure to add the keys as attributes. + schema = self._table_schemas[table] + attributes[schema.hash_key_schema.name] = key.hash_key + if key.range_key is not None: + attributes[schema.range_key_schema.name] = key.range_key + return_attrs = self._UpdateItem(item, attributes, expected, return_values) + result = PutResult(return_values=return_attrs, write_units=1) + return self._HandleCallback(callback, result) + + def DeleteItem(self, table, key, callback, expected=None, + return_values=None): + assert not self._read_only, 'Received "DeleteItem" request on read-only database' + + self._CheckKey(table, key, None, expected) + item = self._GetItem(table, key) + return_attrs = self._UpdateItem(item, None, expected, return_values) + self._GetItem(table, key, delete=True) + result = DeleteResult(return_values=return_attrs, write_units=1) + return self._HandleCallback(callback, result) + + def UpdateItem(self, table, key, callback, attributes, expected=None, return_values=None): + self._CheckKey(table, key, None, expected) + assert not self._read_only, 'Received "UpdateItem" request on read-only database' + + if len(attributes) == 0: + # If an UpdateItem request has a valid key but no additional attributes, DynamoDB returns + # a seemingly positive result but does not actually create the item. We are just emulating + # that behavior here. + result = UpdateResult(return_values={}, write_units=1) + return self._HandleCallback(callback, result) + item = self._GetItem(table, key) + # Make sure to add the keys as attributes. + schema = self._table_schemas[table] + attributes[schema.hash_key_schema.name] = key.hash_key + if key.range_key is not None: + attributes[schema.range_key_schema.name] = key.range_key + return_attrs = self._UpdateItem(item, attributes, expected, return_values) + result = UpdateResult(return_values=return_attrs, write_units=1) + return self._HandleCallback(callback, result) + + def Query(self, table, hash_key, range_operator, callback, attributes, + limit=None, consistent_read=False, count=False, + scan_forward=True, excl_start_key=None): + schema = self._table_schemas[table] + assert schema.range_key_schema, 'schema has no range key' + self._CheckKeyType(table, schema.hash_key_schema, 'hash key', hash_key) + range_dict = self._tables[table].get(hash_key, {}) + keys = sorted(range_dict.keys()) + if count: + assert not attributes, 'cannot specify attributes and count=True' + # TODO(spencer): determine what the read-units ought to be here. + result = QueryResult(count=len(keys), items=[], last_key=None, + read_units=(len(keys) + 1023) / 1024) + return self._HandleCallback(callback, result) + + # Handle range operator. + if range_operator: + key = range_operator.key[0] + if range_operator.op == 'EQ': + i = bisect_left(keys, key) + keys = keys[i:i + 1] if i != len(keys) and keys[i] == key else [] + elif range_operator.op == 'LT': + i = bisect_left(keys, key) + keys = keys[0:i] if i else [] + elif range_operator.op == 'LE': + i = bisect_right(keys, key) + keys = keys[0:i] if i else [] + elif range_operator.op == 'GT': + i = bisect_right(keys, key) + keys = keys[i:] if i != len(keys) else [] + elif range_operator.op == 'GE': + i = bisect_left(keys, key) + keys = keys[i:] if i != len(keys) else [] + elif range_operator.op == 'BEGINS_WITH': + keys = [v for v in keys if v.startswith(key)] + elif range_operator.op == 'BETWEEN': + s = bisect_left(keys, key) + e = bisect_right(keys, range_operator.key[1]) + if s < len(keys) and e > 0: + keys = keys[s:e] + else: + keys = [] + + # Skip everything before (or after) excl_start_key if given. + if excl_start_key is not None: + assert excl_start_key.range_key != '', 'empty start key not supported (same as DynamoDB)' + self._CheckKeyType(table, schema.range_key_schema, 'start key', excl_start_key.range_key) + if scan_forward: + i = bisect_right(keys, excl_start_key.range_key) + keys = keys[i:] if i != len(keys) else [] + else: + i = bisect_left(keys, excl_start_key.range_key) + keys = keys[0:i] if i else [] + # Reverse keys if scanning backwards. + if not scan_forward: + keys.reverse() + # Limit size of results. + if limit is not None and limit < len(keys): + keys = keys[0:limit] + last_key = DBKey(hash_key=hash_key, range_key=keys[-1]) if len(keys) > 0 else None + else: + last_key = None + + bytes_read = 0 + items = [] + for k in keys: + item = self._GetAttributes(range_dict[k], attributes) + if item: + items.append(item) + bytes_read += len(k) if isinstance(k, (str, unicode)) else 8 + bytes_read += sum([len(a) + (len(d) if isinstance(d, (str, unicode)) else 8) for a, d in item.items()]) + + read_units = (bytes_read / (1 if consistent_read else 2) + 1023) / 1024 + result = QueryResult(count=len(keys), items=items, last_key=last_key, read_units=read_units) + return self._HandleCallback(callback, result) + + def Scan(self, table, callback, attributes, limit=None, excl_start_key=None, scan_filter=None): + """Moves sequentially through entire table until 'excl_start_key' + is located. Then iterates, passing each item through the + conditions of 'scan_filter', accumulating up to 'limit' results. + """ + assert limit is None or limit > 0, limit + items = [] + last_key = None + bytes_read = 0 + found = False + + def _FilterItem(item): + """Returns whether the item passes the conditions of + 'scan_filter'. This implementation is incomplete and supports + only a subset of operators. + """ + if scan_filter: + for attr, condition in scan_filter.items(): + if attr not in item: + return False + elif condition.op == 'EQ': + if item[attr] != condition.value[0]: + return False + elif condition.op == 'LT': + if item[attr] >= condition.value[0]: + return False + elif condition.op == 'LE': + if item[attr] > condition.value[0]: + return False + elif condition.op == 'GT': + if item[attr] <= condition.value[0]: + return False + elif condition.op == 'GE': + if item[attr] < condition.value[0]: + return False + elif condition.op == 'BEGINS_WITH': + if item[attr] != condition.value[0]: + return False + elif condition.op == 'BETWEEN': + if item[attr] < condition.value[0] or item[attr] > condition.value[1]: + return False + return True + + for hash_key, value in self._tables[table].items(): + # Iterate until we find last processed exclusive start key hash value. + # These aren't in sorted order, so we just iterate until we find start + # hash key before starting the scan. + if not found and excl_start_key and excl_start_key.hash_key != hash_key: + continue + else: + found = True + # Handle composite-key scan. + if self._table_schemas[table].range_key_schema: + keys = sorted(value.keys()) + if excl_start_key and excl_start_key.hash_key == hash_key: + assert excl_start_key.range_key != '', 'empty start key not supported (same as DynamoDB)' + i = bisect_right(keys, excl_start_key.range_key) + keys = keys[i:] if i != len(keys) else [] + for key in keys: + bytes_read += sum([len(a) + (len(d) if isinstance(d, (str, unicode)) else 8) for a, d in value[key].items()]) + if _FilterItem(value[key]): + item = self._GetAttributes(value[key], attributes) + if item: + items.append(item) + if len(items) == limit: + last_key = DBKey(hash_key=hash_key, range_key=key) + break + else: + if not excl_start_key or excl_start_key.hash_key != hash_key: + bytes_read += sum([len(a) + (len(d) if isinstance(d, (str, unicode)) else 8) for a, d in value.items()]) + if _FilterItem(value): + item = self._GetAttributes(value, attributes) + if item: + items.append(item) + if limit is not None and len(items) == limit: + if hash_key != self._tables[table].keys()[-1]: + last_key = DBKey(hash_key=hash_key, range_key=None) + break + + read_units = (bytes_read / 2 + 1023) / 1024 + result = ScanResult(count=len(items), items=items, last_key=last_key, read_units=read_units) + return self._HandleCallback(callback, result) + + def AddTimeout(self, deadline_secs, callback): + """Invokes the specified callback after 'deadline_secs'.""" + return IOLoop.current().add_timeout(time.time() + deadline_secs, callback) + + def AddAbsoluteTimeout(self, abs_timeout, callback): + """Invokes the specified callback at time 'abs_timeout'.""" + return IOLoop.current().add_timeout(abs_timeout, callback) + + def RemoveTimeout(self, timeout): + """Removes an existing timeout.""" + IOLoop.current().remove_timeout(timeout) + + def _CheckKey(self, table, key, must_exist, expected): + """Verifies the key matches the key schema.""" + assert table in self._table_schemas, 'table %s does not exist in %r' % (table, self._table_schemas) + schema = self._table_schemas[table] + + if expected and expected.has_key(schema.hash_key_schema.name): + assert expected[schema.hash_key_schema.name] == False + must_exist = False + + assert key.hash_key is not None, 'need hash key: %s' % repr(key) + self._CheckKeyType(table, schema.hash_key_schema, 'hash key', key.hash_key) + if must_exist == True: + assert key.hash_key in self._tables[table], 'key %s does not exist' % key.hash_key + elif must_exist == False and schema.range_key_schema == None: + if key.hash_key in self._tables[table]: + raise DBConditionalCheckFailedError('key %s already exists' % key.hash_key) + + if key.range_key is not None: + assert schema.range_key_schema is not None, 'table has no range key in schema' + self._CheckKeyType(table, schema.range_key_schema, 'range key', key.range_key) + + if must_exist == True: + assert key.range_key in self._tables[table][key.hash_key], \ + 'range key %s does not exist' % key.range_key + elif must_exist == False and key.hash_key in self._tables[table]: + if key.range_key in self._tables[table][key.hash_key]: + raise DBConditionalCheckFailedError('range key %s already exists' % key.range_key) + + else: + assert schema.range_key_schema is None, 'missing range key' + + def _CheckKeyType(self, table, key_schema, key_type, value): + """Ensures that 'value' is of a type compatible with 'key_schema' + (which is either 'N' for number or 'S' for string). + """ + if key_schema.value_type == 'N': + assert isinstance(value, (int, long, float)), \ + '%s for column "%s" in table "%s" must be a number: %s' % (key_type, key_schema.name, table, repr(value)) + elif key_schema.value_type == 'S': + assert isinstance(value, (str, unicode)), \ + '%s for column "%s" in table "%s" must be a string: %s' % (key_type, key_schema.name, table, repr(value)) + else: + assert False, 'unexpected schema type "%s"' % key_schema.schema_type + + def _NewSchemaStatus(self, schema, new_status): + """Returns a new schema with status set to 'new_status'.""" + return TableSchema(create_time=schema.create_time, + hash_key_schema=schema.hash_key_schema, + range_key_schema=schema.range_key_schema, + read_units=schema.read_units, + write_units=schema.write_units, + status=new_status) + + def _GetItem(self, table, key, delete=False): + """Fetches the item from the store by table & key. If 'delete', + deletes the item. + """ + if key.range_key is not None: + if key.hash_key not in self._tables[table]: + self._tables[table][key.hash_key] = dict() + if key.range_key not in self._tables[table][key.hash_key]: + self._tables[table][key.hash_key][key.range_key] = dict() + if not delete: + return self._tables[table][key.hash_key][key.range_key] + else: + del self._tables[table][key.hash_key][key.range_key] + else: + if not delete: + if key.hash_key not in self._tables[table]: + self._tables[table][key.hash_key] = dict() + return self._tables[table][key.hash_key] + else: + del self._tables[table][key.hash_key] + + def _GetAttributes(self, item, attributes): + """Gets the list of named 'attributes' from the item. If an + attribute is not present in the item, it won't be returned. + If attributes is None, all attributes are returned. + """ + if not attributes: + return item + result = dict([(a, item[a]) for a in attributes if item.has_key(a)]) + return result + + def _UpdateItem(self, item, attributes, expected, return_values): + """Logic to update a data item. Called from PutItem() and + UpdateItem(). + """ + if expected: + for k, v in expected.items(): + if isinstance(v, bool): + assert v == False + if item.has_key(k): + raise DBConditionalCheckFailedError('expected attr %s not to exist, but exists with value %r' + % (k, item[k])) + else: + if not item.has_key(k): + raise DBConditionalCheckFailedError('expected attr %s does not exist: %r' % (k, item)) + if item[k] != v: + raise DBConditionalCheckFailedError('expected mismatch: %s != %s' % (repr(item[k]), repr(v))) + return_attrs = None + if return_values == 'ALL_OLD': + return_attrs = copy.deepcopy(item) + elif return_values == 'UPDATED_OLD': + if not attributes: + return_attrs = copy.deepcopy(item) + else: + return_attrs = dict([(k, copy.deepcopy(item[k])) for k in attributes.keys()]) + + # Update (or delete) the item. + if attributes: + for key, update in attributes.items(): + if isinstance(update, UpdateAttr): + if update.action == 'PUT': + assert not isinstance(update.value, list) or update.value, \ + 'PUT of empty list is not supported (matches DynamoDB behavior)' + item[key] = update.value + elif update.action == 'ADD': + if isinstance(update.value, list): + if not key in item: + item[key] = list() + for v in update.value: + if v not in item[key]: + item[key].append(v) + item[key].sort() + else: + assert isinstance(update.value, (int, long, float)), \ + 'value not number: %s' % repr(update.value) + if not key in item: + item[key] = 0 + item[key] += update.value + elif update.action == 'DELETE': + if isinstance(update.value, list): + for v in update.value: + if key in item and v in item[key]: + item[key].remove(v) + elif key in item: + # Delete attribute in item, if it exists (matches DynamoDB behavior). + del item[key] + else: + item[key] = update + else: + item.clear() + + if return_values == 'ALL_NEW': + return_attrs = copy.deepcopy(item) + elif return_values == 'UPDATED_NEW': + if not attributes: + return_attrs = dict() + else: + return_attrs = dict([(k, copy.deepcopy(item[k])) for k in attributes.keys()]) + + return return_attrs + + def _HandleCallback(self, callback, result): + """If callback is not None, runs asynchronously; otherwise, runs + synchronously. + """ + if any(isinstance(result, rt) for rt in LocalClient._MUTATING_RESULTS): + self._persist.MarkDirty() + if callback: + IOLoop.current().add_callback(partial(callback, result)) + else: + return result + + def _GetTableSize(self, table): + """Computes the number of elements in the table. If the table + schema has a composite key, iterates over each hash key to compute + the full length. + """ + schema = self._table_schemas[table] + if schema.range_key_schema: + return sum([len(rd) for rd in self._tables[table].values()]) + else: + return len(self._tables[table]) diff --git a/backend/db/local_persist.py b/backend/db/local_persist.py new file mode 100644 index 0000000..5b1332b --- /dev/null +++ b/backend/db/local_persist.py @@ -0,0 +1,149 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Persistence for local DB. + +If --localdb_dir=<> is specified, then the in-memory python data is +persisted to disk every --localdb_sync_secs=<> seconds. + +PERIODIC PERSISTENCE: the in-memory python data structures +representing the local database are persisted to disk every +--localdb_sync_secs with a corresponding fsync call. Each successive +sync requires that the previous sync has completed. The on-disk sync +are synchronous and all other processing in the server will come to a +halt during this period. Each sync is written to the current run's +filename plus a '.sync' suffix. Upon completion and fsync, the .sync +file is renamed to the original in an atomic step. + +SUCCESSIVE RUNS: upon restart the server looks in --localdb_dir for +the most recent, fully-written datastore persistence file. These files +are named "viewfinder.db.0". There are at most 5 recent versions of +the database, starting with no suffix and ending with ".4". On +startup, the current set of files are 'rolled'. The file with suffix +".4" is deleted; the file with suffix ".3" is moved to ".4", +etc. During a run, only the file with ".0" is updated. + +A specific version of the database may be used instead of ".0" as the +current on a new run by specifying --localdb_version=<>. + +Specify --localdb_reset to clear all old versions and start from +scratch. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import os +import cPickle as pickle +import shutil +import re +import time + +from tornado import ioloop, options + + +class DBPersist(object): + """Local datastore persistence for extended testing scenarios that + must survive server restarts. + """ + _BASE_NAME = 'server.db' + + def __init__(self, tables, table_schemas): + """Sets up a periodic callback for sync operations.""" + self._tables = tables + self._table_schemas = table_schemas + self._is_dirty = False + self._db_dir = options.options.localdb_dir + if options.options.localdb_dir: + logging.info('enabling local datastore persistence') + self._sync_callback = ioloop.PeriodicCallback( + self._DBSync, options.options.localdb_sync_secs * 1000) + self._sync_callback.start() + # Initialize output directory and files. + self._InitFiles() + + def Shutdown(self): + """Does a final sync on shutdown.""" + self._DBSync() + + def MarkDirty(self): + """Called by the local datastore when its contents have been + modified and another sync should be scheduled. + """ + self._is_dirty = True + + def _InitFiles(self): + """Initializes the output directory and output files. The selected + previous version is copied to '.0.sync', and versions from + previous runs are rolled. '.0.sync' is then renamed to + '.0'. + """ + # Verify / create db output directory. + try: + files = os.listdir(self._db_dir) + except: + files = [] + os.makedirs(self._db_dir) + + if files and options.options.localdb_reset: + logging.warning('resetting local datastore persistence') + + version_re = re.compile(r'%s.([0-9]+)$' % DBPersist._BASE_NAME) + versions = dict() + for f in files: + match = version_re.match(f) + if match: + if options.options.localdb_reset: + os.unlink(os.path.join(self._db_dir, f)) + else: + versions[int(match.group(1))] = f + + # Do a hard-link shuffle to rotate files. + def _GetPath(v): + return os.path.join(self._db_dir, '%s.%d' % (DBPersist._BASE_NAME, v)) + + use_version = options.options.localdb_version + srcs = [_GetPath(use_version)] + dsts = [_GetPath(0) + '.sync'] + for i in xrange(options.options.localdb_num_versions - 1, 0, -1): + srcs.append(_GetPath(i - 1)) + dsts.append(_GetPath(i)) + for src, dst in zip(srcs, dsts): + if os.access(dst, os.W_OK): os.unlink(dst) + if os.access(src, os.W_OK): os.link(src, dst) + + self._cur_file = _GetPath(0) + self._tmp_file = _GetPath(0) + '.sync' + if os.access(self._cur_file, os.W_OK): os.unlink(self._cur_file) + if os.access(self._tmp_file, os.W_OK): + shutil.copyfile(self._tmp_file, self._cur_file) + os.unlink(self._tmp_file) + + # Initialize the database from the source database file. + if os.access(self._cur_file, os.W_OK): + logging.info('initializing from persisted db file %s...' % self._cur_file) + start_time = time.time() + with open(self._cur_file, 'r') as f: + (tables, schemas) = pickle.load(f) + self._tables.update(tables) + self._table_schemas.update(schemas) + logging.info('initialization took %.4fs' % (time.time() - start_time)) + + def _DBSync(self): + """Periodic callback for data persistence. + """ + if not self._is_dirty: + return + self._is_dirty = False + assert self._cur_file, self._tmp_file + + logging.info('syncing local datastore...') + start_time = time.time() + with open(self._tmp_file, 'w') as f: + pickle.dump((self._tables, self._table_schemas), f, pickle.HIGHEST_PROTOCOL) + os.fsync(f) + + if os.access(self._cur_file, os.W_OK): os.unlink(self._cur_file) + os.link(self._tmp_file, self._cur_file) + assert os.access(self._cur_file, os.W_OK) + + logging.info('sync took %.4fs' % (time.time() - start_time)) diff --git a/backend/db/lock.py b/backend/db/lock.py new file mode 100644 index 0000000..3b6c7a8 --- /dev/null +++ b/backend/db/lock.py @@ -0,0 +1,411 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Lock description. + +A lock is "acquired" on a "resource" in order to control concurrent +access to that resource. Resources are grouped by "resource type", and +within each type, resources are distinguished by a unique identifier. +In addition, resource-specific data can be stored in the lock. Each +lock has a single owner, who is the only one that is allowed to modify +the resource during the lock's lifetime. + +At this time, locks are assumed to govern write access to resources. +They do not restrict read access. Therefore, a resource may be read +even while a lock has been acquired on it by another agent. + +Locks are stored in the database so that they can survive process failure. +If a lock is acquired by an owner, and the owner fails in such a way +that the lock is never released, then the lock has been "abandoned". +Each lock owner must guarantee that every lock acquired is also released. +In order to assist owners in meeting this guarantee, owners can request +"abandonment detection" when acquiring a lock. Locks with abandonment +detection will be tagged with an expiration attribute. As long as the +acquiring process has not failed, the expiration will be updated on +a periodic basis in order to "renew" the lock. If the lock is not +renewed for a time, the lock is considered to be abandoned, and will +be cleaned up by a periodic sweep of the lock table. + +Note that even though the Lock class can *detect* that a lock has been +abandoned, it cannot actually *release* those locks, as only code with +specific knowledge of a particular resource type can do that safely. + +TODO(Andy): Add a LockManager class which offers callers a way to + register for notifications when locks of a particular + resource type have been abandoned. The LockManager would + periodically scan the Lock table to find abandoned locks. + + Lock: control concurrent access to resources +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging +import random +import sys +import time + +from datetime import timedelta +from functools import partial +from tornado import gen +from tornado.ioloop import IOLoop +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import LockFailedError +from viewfinder.backend.db import vf_schema, db_client +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject + +@DBObject.map_table_attributes +class Lock(DBHashObject): + """Lock object. The "Lock.TryAcquire" method acquires a lock on an + instance of some resource type. If the acquire succeeds, then the + lock owner has sole write access to the resource for the lifetime + of the lock. The Release method *must* eventually be called by the + owner, even in cases of process failure. The TryAcquire method has + a "detect_abandonment" option that uses the lock "expiration" + attribute and a periodic renewal to detect cases where the owner + has failed and will not be able to call Release. + """ + __slots__ = ['_unique_id', '_is_released', '_timeout', '_renewing'] + + _table = DBObject._schema.GetTable(vf_schema.LOCK) + + FAILED_TO_ACQUIRE_LOCK = 0 + ACQUIRED_LOCK = 1 + ACQUIRED_ABANDONED_LOCK = 2 + + MAX_UPDATE_ATTEMPTS = 10 + """Limit of attempts to acquire the lock in case where race conditions + or bugs force repeated attempts. + """ + + ABANDONMENT_SECS = 60.0 + """Locks with abandonment detection are assumed to be abandoned after 60 + seconds. + """ + + LOCK_RENEWAL_SECS = 30.0 + """Locks with abandonment detection are renewed every 30 seconds, giving + a 30 second window for slow renewals. + """ + + def __init__(self, lock_id=None, owner_id=None): + super(Lock, self).__init__() + self.lock_id = lock_id + self._GenerateOwnerId() if owner_id is None else self._SetOwnerId(owner_id) + self._is_released = False + self._timeout = None + self._renewing = False + + @classmethod + def ConstructLockId(cls, resource_type, resource_id): + """A lock id is the concatenation of the resource type and resource + id, separated by a colon. For example: + op:123 + vp:v--F + """ + assert resource_type and ":" not in resource_type, resource_type + assert resource_id and ":" not in resource_id, resource_id + return resource_type + ':' + resource_id + + @classmethod + def DeconstructLockId(cls, lock_id): + """Returns the components of a lock identifier: + (resource_type, resource_id) + """ + index = lock_id.find(':') + assert index != -1 + return lock_id[:index], lock_id[index + 1:] + + @classmethod + def TryAcquire(cls, client, resource_type, resource_id, callback, + resource_data=None, detect_abandonment=False, owner_id=None): + """Tries to acquire a lock on the specific resources instance, + associating an optional "resource_data" string with the lock. + Returns a tuple containing the lock object and a status value: + (lock, status) + + The status value is one of: + FAILED_TO_ACQUIRE_LOCK + TryAcquire was unable to acquire the lock because another + agent has already acquired it. In this case, the lock's + "acquire_failures" attribute is incremented. + + ACQUIRED_LOCK + TryAcquire successfully acquired the lock. + + ACQUIRED_ABANDONED_LOCK + TryAcquire successfully acquired the lock, but the lock had + been abandoned by the previous owner. In this case, the lock's + resource_data is set to the previous owner's value rather than + the new value. The new owner should ensure that the resource + is in a complete and consistent state before proceeding. + + If the "detect_abandonment" option is set, then uses the lock's + "expiration" attribute as a "heartbeat" to detect failure of this + process. If the expiration is not continually renewed, then the + lock will expire and be considered abandoned. + """ + Lock._TryAcquire(client, resource_type, resource_id, callback, + resource_data=resource_data, detect_abandonment=detect_abandonment, + owner_id=owner_id) + + @classmethod + @gen.engine + def Acquire(cls, client, resource_type, resource_id, owner_id, callback): + """Acquires lock or fails with LockFailedError. + Returns lock as only parameter to callback. + """ + results = yield gen.Task(Lock.TryAcquire, client, resource_type, resource_id, owner_id=owner_id) + lock, status = results.args + if status == Lock.FAILED_TO_ACQUIRE_LOCK: + raise LockFailedError('Cannot acquire lock "%s:%s", owner_id "%s" because another agent has acquired it' % + (resource_type, resource_id, owner_id)) + callback(lock) + + @classmethod + def ScanAbandoned(cls, client, callback, limit=None, excl_start_key=None): + """Scans the Lock table for locks that have expired, and therefore + are assumed to have been abandoned by their owners. Returns a tuple + containing a list of abandoned locks and the key of the last lock + that was scanned (or None if all locks have been scanned). + """ + assert limit > 0, limit + now = int(time.time()) + Lock.Scan(client, None, callback, limit=limit, + excl_start_key=excl_start_key, + scan_filter={'expiration': db_client.ScanFilter([now], 'LE')}) + + def IsAbandoned(self): + """Returns true if this lock has been abandoned, which means that the + process which acquired it failed and will never release it. Abandonment + is assumed to have occurred if enough time has passed since the last + renewal. + """ + return self.expiration is not None and self.expiration <= time.time() + + def AmOwner(self): + """Returns true if the lock is owned by the current instance.""" + return self._unique_id == self.owner_id + + def IsReleased(self): + """Returns true if the lock has been released via a call to "Release".""" + return self._is_released + + @gen.coroutine + def Release(self, client): + """Releases the lock so that it may be acquired by other agents. Deletes + the lock from the Lock table. In addition, updates the "acquire_failures" + attribute on the lock to the value in the database. This attribute allows + the releasing owner to see whether any other agents tried to acquire the + lock during the period in which it was held. + """ + assert not self.IsAbandoned(), self + assert self.AmOwner(), self + assert not self.IsReleased(), self + + self._StopRenewal() + + do_raise = False + + try: + expected_acquire_failures = False if self.acquire_failures is None else self.acquire_failures + yield gen.Task(self.Delete, client, expected={'acquire_failures': expected_acquire_failures, + 'owner_id': self.owner_id}) + except Exception: + type, value, tb = sys.exc_info() + logging.warning('release of "%s" lock failed (will retry): %s' % (self, value)) + lock = yield gen.Task(Lock.Query, client, self.lock_id, None) + # As long as we still have ownership of lock, update acquire_failures and retry. + if lock.owner_id == self.owner_id: + self.acquire_failures = lock.acquire_failures + yield self.Release(client) + raise gen.Return() + else: + # Shouldn't happen, but we definitely want to know if it is happening. + resource_type, resource_id = Lock.DeconstructLockId(self.lock_id) + raise LockFailedError('Cannot release lock "%s:%s", owner_id "%s" because another agent, owner_id "%s", ' + 'owns it' % + (resource_type, resource_id, self.owner_id, lock.owner_id)) + + self._is_released = True + + def Abandon(self, client, callback): + """Marks the lock as abandoned by disabling renewal and expiring the + lock. Other agents may acquire or release the lock, but must first + be certain that the protected resource is in a consistent state. + """ + assert self.AmOwner(), self + self._StopRenewal() + self.expiration = 0 + self.Update(client, expected={'owner_id': self.owner_id}, callback=callback) + + @classmethod + def _TryAcquire(cls, client, resource_type, resource_id, callback, + resource_data=None, detect_abandonment=False, owner_id=None, + attempts=0, test_hook=None): + """Helper method that makes "MAX_UPDATE_ATTEMPTS" to acquire the + lock. If multiple agents are trying to acquire the lock, then one + might be updating the lock in order to acquire it while another + is querying the lock in order to see if it can be acquired. These + race conditions are resolved by detecting changes and retrying. + + The "test_hook" function is called just before an attempt is made + to update the Lock row in the database. This makes testing various + race conditions much easier. + """ + def _OnUpdate(lock, status): + """If lock was acquired and abandonment needs to be detected, + then starts renewal timer. + """ + if status != Lock.FAILED_TO_ACQUIRE_LOCK and detect_abandonment: + lock._Renew(client) + + callback(lock, status) + + def _OnException(type, value, tb): + """Starts over unless too many acquire attempts have been made.""" + logging.warning('race condition caused update of "%s:%s" lock to fail (will retry): %s' % + (resource_type, resource_id, value)) + + # Retry the acquire unless too many attempts have already been made. + if attempts >= Lock.MAX_UPDATE_ATTEMPTS: + logging.error('too many failures attempting to update lock; aborting', exc_info=(type, value, tb)) + callback(None, Lock.FAILED_TO_ACQUIRE_LOCK) + else: + # TODO(Andy): We really should consider adding in a randomly-perturbed, exponential backoff + # here to avoid senseless fights between concurrent servers. + Lock._TryAcquire(client, resource_type, resource_id, callback, + resource_data=resource_data, detect_abandonment=detect_abandonment, + attempts=attempts + 1, test_hook=test_hook) + + def _DoUpdate(update_func): + """Calls the test hook just before a row is updated in the database. + Tests can simulate updates made by another agent at this critical + juncture. + """ + if test_hook is not None: + test_hook(update_func) + else: + update_func() + + def _OnQuery(lock): + """Creates a new lock, takes control of an abandoned lock, or reports + failure to acquire the lock. The choice depends upon the current + state of the lock in the database. Other agents may be trying to + acquire the lock at the same time, so takes care to handle those race + conditions. + """ + if lock is None: + # Create new lock. + lock = Lock(lock_id, owner_id) + if resource_data is not None: + lock.resource_data = resource_data + if detect_abandonment: + lock.expiration = time.time() + Lock.ABANDONMENT_SECS + + with util.Barrier(partial(_OnUpdate, lock, Lock.ACQUIRED_LOCK), on_exception=_OnException) as b: + _DoUpdate(partial(lock.Update, client, expected={'lock_id': False}, callback=b.Callback())) + elif lock._IsOwnedBy(owner_id): + assert not detect_abandonment, (resource_type, resource_data) + # Acquirer knew owner id, so sync _unique_id up with it. + lock._SyncUniqueIdToOwnerId() + _OnUpdate(lock, Lock.ACQUIRED_LOCK) + elif lock.IsAbandoned(): + logging.warning('lock was abandoned; trying to take control of it: %s' % lock) + + # Try to take control of lock. + with util.Barrier(partial(_OnUpdate, lock, Lock.ACQUIRED_ABANDONED_LOCK), on_exception=_OnException) as b: + _DoUpdate(partial(lock._TryTakeControl, client, detect_abandonment, b.Callback())) + else: + logging.warning('acquire of lock failed; already held by another agent: %s' % lock) + + # Report the failure to acquire in order to track contention on the lock and to notify the current + # lock owner that another agent tried to acquire the lock. + with util.Barrier(partial(_OnUpdate, None, Lock.FAILED_TO_ACQUIRE_LOCK), on_exception=_OnException) as b: + _DoUpdate(partial(lock._TryReportAcquireFailure, client, b.Callback())) + + # Get current state of the lock from the database. + lock_id = Lock.ConstructLockId(resource_type, resource_id) + Lock.Query(client, lock_id, None, _OnQuery, must_exist=False) + + def _StopRenewal(self): + """Stops renewing this lock's expiration on a periodic basis.""" + if self._timeout is not None: + IOLoop.current().remove_timeout(self._timeout) + self._timeout = None + self._renewing = False + + def _IsOwnedBy(self, owner_id): + """Compares against Lock.owner_id.""" + assert self.owner_id is not None # None should never match self.owner_id + return owner_id == self.owner_id + + def _SetOwnerId(self, owner_id): + """Set owner_id and asserts that owner_id argument is not None.""" + assert owner_id is not None + self._unique_id = owner_id + self.owner_id = owner_id + + def _SyncUniqueIdToOwnerId(self): + """We matched expected owner_id to actual owner_id. Now, get _unique_id into sync with owner_id.""" + self._unique_id = self.owner_id + + def _GenerateOwnerId(self): + """Generates a random 48 bit number (converted to string) which is used as the owner id. + This string is a decimal representation of a 48 bit random number and not 6 random bytes. + """ + self._SetOwnerId(str(random.getrandbits(48))) + + def _TryTakeControl(self, client, detect_abandonment, callback): + """Sets new owner on this lock and update expiration.""" + former_owner_id = self.owner_id + self._GenerateOwnerId() + self.expiration = time.time() + Lock.ABANDONMENT_SECS if detect_abandonment else None + self.Update(client, expected={'owner_id': former_owner_id}, callback=callback) + + def _TryReportAcquireFailure(self, client, callback): + """Increments the "acquire_failures" attribute on the lock. Multiple + agents may concurrently try to acquire the lock, so this operation + must handle race conditions. Expecting the owner_id is a way to ensure + that the lock hasn't been released by the owner before we update it here. + Otherwise, our attempt to update a released lock will result in a lock row without + expiration or resource_data fields which means a stuck lock. + """ + expected = {'owner_id': self.owner_id} + + if self.acquire_failures is None: + self.acquire_failures = 1 + expected['acquire_failures'] = False + else: + self.acquire_failures += 1 + expected['acquire_failures'] = self.acquire_failures - 1 + + self.Update(client, expected=expected, callback=callback) + + def _Renew(self, client): + """Continually renews the lock by updating its expiration on a regular + interval. As long as the expiration is in the future, the lock is not + considered to be abandoned. + """ + def _OnException(type, value, tb): + """If failure occurs during renewal, just abandon the lock.""" + logging.error('failure trying to renew lock "%s"', exc_info=(type, value, tb)) + + def _OnRenewalTimeout(): + self._timeout = None + if not self._renewing: + return + logging.info('renewing lock: %s' % self) + + with util.Barrier(_ScheduleNextRenew, on_exception=_OnException) as b: + self.expiration = time.time() + Lock.ABANDONMENT_SECS + self.Update(client, b.Callback()) + + def _ScheduleNextRenew(): + if not self._renewing: + return + self._timeout = IOLoop.current().add_timeout(timedelta(seconds=Lock.LOCK_RENEWAL_SECS), _OnRenewalTimeout) + + self._renewing = True + _ScheduleNextRenew() diff --git a/backend/db/lock_resource_type.py b/backend/db/lock_resource_type.py new file mode 100644 index 0000000..c004a23 --- /dev/null +++ b/backend/db/lock_resource_type.py @@ -0,0 +1,24 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Enumeration of lock resource types. + +Locks are acquired on resources. Each resource has a type and an id. The +resource type needs to be a unique string that ensures there is no conflict +between locks of different types. This enumeration lists the resource types +for locks taken by the Viewfinder backend. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + + +class LockResourceType: + """Set of resource types used for locking. Each lock resource type should + be two or three characters that are different from all other types. The + resource type is concatenated with the resource id to form the lock id. + For example: + op:123 + vp:v--F + """ + Job = 'job' # Resource id is job name (dbchk, get_logs, etc...). + Operation = 'op' # Resource id is user that initiated operation. + Viewpoint = 'vp' # Resource id is the viewpoint id. diff --git a/backend/db/metric.py b/backend/db/metric.py new file mode 100644 index 0000000..31405f7 --- /dev/null +++ b/backend/db/metric.py @@ -0,0 +1,300 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder performance metrics. Metrics are captured from a machine +using the counters module, serialized, and stored using this table. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import json +import logging +import time +import platform +from collections import namedtuple +from functools import partial + +from tornado.ioloop import IOLoop +from viewfinder.backend.base import util, counters, retry +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + + +# Default clustering key for metrics. The cluster name is used in combination with +# additional information to create a group key for metrics. Use of this variable +# should be considered temporary until a more comprehensive deployment model is +# developed. +DEFAULT_CLUSTER_NAME = 'us-east-1' + +# Group name for daily logs analysis output. +LOGS_STATS_NAME = 'logs' + +# Group name for metrics written by periodic jobs (dbchk, logs merge, logs analysis). +JOBS_STATS_NAME = 'jobs' + +# A metric interval is a combination of a name and a length of time in seconds. +# The name is intended for use as part of the group_key when creating metric +# object. +MetricInterval = namedtuple('MetricInterval', ['name', 'length']) + + +# Configured metric collection intervals. Each interval is a tuple (name, frequency) where frequency is +# specified in seconds. If additional intervals are added, this list should be maintained in ascending +# order by interval length. +METRIC_INTERVALS = [MetricInterval('detail', 30), MetricInterval('hourly', 3600)] +LOGS_INTERVALS = [MetricInterval('daily', 86400)] +JOBS_INTERVALS = [MetricInterval('daily', 86400)] + +def GetMachineKey(): + """Gets the machine key to be used for Metrics uploaded from this process.""" + return platform.node() + + +class MetricUploadRetryPolicy(retry.RetryPolicy): + """Retry policy for uploading Metrics to the server.""" + def __init__(self, max_tries=3, timeout=30, min_delay=.5, max_delay=5): + retry.RetryPolicy.__init__(self, max_tries=max_tries, timeout=timeout, + min_delay=min_delay, max_delay=max_delay, + check_result=None, check_exception=self._ShouldRetry) + + def _ShouldRetry(self, type_, value_, traceback): + """Stub retry method, indicating that all exceptions should result in a retry.""" + return True + + +@DBObject.map_table_attributes +class Metric(DBRangeObject): + """Viewfinder metric data object. A Metric object represents the collected performance + counters from one Viewfinder server instance at a specific point in time. The metric + is additionally associated with a group key, which helps to organize metrics collected + from several different machines into a more natural group for querying. + """ + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.METRIC) + _timeouts = dict() + + def __init__(self, group_key=None, sort_key=None): + """Initialize a new Metric object.""" + super(Metric, self).__init__() + self.group_key = group_key + self.sort_key = sort_key + + @classmethod + def QueryTimespan(cls, client, group_key, start_time, end_time, callback, excl_start_key=None): + """Performs a range query on the metrics table for the given group_key the given start and + end time. An optional start key can be specified to resume an earlier query which did not + retrieve the full result set. + Either start_time or end_time may be None, but not both. + """ + # Query from start_time to end_time + 1 - because they contain a machine id, actual + # sort keys will always follow a key comprised only of a timestamp. + assert start_time is not None or end_time is not None, 'must specify at least one of start_time and end_time' + operator = None + start_rk = util.CreateSortKeyPrefix(start_time, randomness=False) if start_time is not None else None + end_rk = util.CreateSortKeyPrefix(end_time + 1, randomness=False) if end_time is not None else None + if start_time is None: + operator = db_client.RangeOperator([end_rk], 'LE') + elif end_time is None: + operator = db_client.RangeOperator([start_rk], 'GE') + else: + operator = db_client.RangeOperator([start_rk, end_rk], 'BETWEEN') + Metric.RangeQuery(client, group_key, operator, None, None, + callback=callback, excl_start_key=excl_start_key) + + @classmethod + def Create(cls, group_key, machine_id, timestamp, payload): + """Create a new metric object with the given attributes. The sort key is + computed automatically from the timestamp and machine id. + """ + sort_key = util.CreateSortKeyPrefix(timestamp, randomness=False) + machine_id + metric = Metric(group_key, sort_key) + metric.machine_id = machine_id + metric.timestamp = timestamp + metric.payload = payload + return metric + + @classmethod + def StartMetricUpload(cls, client, cluster_name, interval): + """Starts an asynchronous loop which periodically samples performance counters and saves + their values to the database. The interval parameter is a MetricInterval object which + specifies the frequency of upload in seconds. + """ + retry_policy = MetricUploadRetryPolicy() + machine_id = GetMachineKey() + meter = counters.Meter(counters.counters) + group_key = cls.EncodeGroupKey(cluster_name, interval) + frequency_seconds = interval.length + + def _UploadError(type_, value_, traceback): + logging.getLogger().error('Unable to upload metrics payload for group: %s' % group_key) + + def _UploadSuccess(): + logging.getLogger().debug('Uploaded metrics payload for group: %s' % group_key) + + def _SnapMetrics(deadline): + """ Method which takes a single sample of performance counters and attempts + to upload the data to the database. This method automatically schedules itself + again on the current IOLoop. + """ + next_deadline = deadline + frequency_seconds + callback = partial(_SnapMetrics, next_deadline) + cls._timeouts[group_key] = IOLoop.current().add_timeout(next_deadline, callback) + + sample = meter.sample() + sample_json = json.dumps(sample) + new_metric = Metric.Create(group_key, machine_id, deadline, sample_json) + with util.Barrier(_UploadSuccess, _UploadError) as b: + retry.CallWithRetryAsync(retry_policy, new_metric.Update, client=client, callback=b.Callback()) + + # Initial deadline should fall exactly on an even multiple of frequency_seconds. + initial_deadline = time.time() + frequency_seconds + initial_deadline -= initial_deadline % frequency_seconds + callback = partial(_SnapMetrics, initial_deadline) + + cls.StopMetricUpload(group_key) + cls._timeouts[group_key] = IOLoop.current().add_timeout(initial_deadline, callback) + + @classmethod + def StopMetricUpload(cls, group_key): + """Stops the metrics upload process if it has already been started. This method is idempotent.""" + if cls._timeouts.get(group_key, None) is not None: + IOLoop.current().remove_timeout(cls._timeouts[group_key]) + cls._timeouts[group_key] = None + + @classmethod + def EncodeGroupKey(cls, cluster_name, interval): + """Encodes a group key for the Metric table. A group key is a combination of a machine cluster + name and a collection interval name. + """ + return cluster_name + '.' + interval.name + + @classmethod + def DecodeGroupKey(cls, group_key): + """Attempts to decode a metrics group key. Returns the machine cluster name and metric + interval used to encode the group. + """ + index = group_key.find('.') + assert index != -1 + cluster = group_key[:index] + interval = group_key[index + 1:] + return cluster, cls.FindIntervalForCluster(cluster, interval) + + @classmethod + def FindIntervalForCluster(cls, cluster, interval): + """Look for and return 'interval' for 'cluster'. Returns None if not found.""" + intervals = [] + + if cluster == DEFAULT_CLUSTER_NAME: + intervals = METRIC_INTERVALS + elif cluster == LOGS_STATS_NAME: + intervals = LOGS_INTERVALS + elif cluster == JOBS_STATS_NAME: + intervals = JOBS_INTERVALS + for i in intervals: + if i.name == interval: + return i + return None + + +class AggregatedMetric(object): + """This is a utility class intended to average multiple samples from a set of several counters + taken over multiple machines. The class is initialized with a start and end time, along with + a set of counters (specified as a subset of counters.counters or another counter collection.) + AggregatedMetric will maintain an internal AggregatedCounter object for each counter specified + in its target counter set. + + Metric objects loaded from the database are added to the metric sequentially - metrics MUST be + added in chronological order by timestamp. The component counter values within each metric sample + will be added to the AggregatedCounter objects. After aggregation, the AggregatedCounter objects + are available from the counter_data member, which is a dictionary keyed by counter name. + + This class is not intended to be instantiated directly - rather, it should be created using + the CreateAggregateForTimespan class method, which automatically handles the details + of querying the backend and aggregating metrics in an efficient way. + """ + class AggregatedCounter(object): + """A utility class used to aggregate data from multiple samples of a single counter, which + may be taken over multiple machines. An AggregatedCounter is initialized by passing it + a counter object - individual samples of the counter should then be added using the AddSample + methods. Samples MUST be added in chronological order by timestamp. + + After samples are added, the following data points are available from this object: + + .machine_data: A dictionary containing the samples collected from each individual machine. + Each member of the dictionary is a list of [timestamp, value] data points. + .cluster_total: A list of [timestamp, value] data points. Each value is the sum of all sample + values for all machines with the same timestamp. + .cluster_avg: An aggregated list like cluster_total, but provides the average value across + all machines rather than a sum. + + The 'is_average' property indicates if this counter's units are an average with a non-time base, + and thus implies that the units of cluster_total will not be analogous to the units of cluster_total. + An example of this would be a counter for the average time per request - this should be averaged + across machines, rather than totaled, because the base of the average is specific to each machine. + """ + def __init__(self, counter): + self.name = counter.name + self.description = counter.description + self.is_average = isinstance(counter, counters._AverageCounter) + self.machine_data = dict() + self.cluster_total = list() + self.cluster_avg = list() + + def AddSample(self, machine, timestamp, value): + """Adds a single sample to the aggregation.""" + self.machine_data.setdefault(machine, list()).append([timestamp, value]) + if len(self.cluster_total) == 0 or timestamp > self.cluster_total[-1][0]: + self.cluster_total.append([timestamp, 0]) + self.cluster_avg.append([timestamp, 0]) + self.cluster_total[-1][1] += value + self.cluster_avg[-1][1] = self.cluster_total[-1][1] / float(len(self.machine_data)) + + + def __init__(self, group_key, start_time, end_time, counter_set): + self.start_time = start_time + self.end_time = end_time + self.group_key = group_key + self.machines = set() + self.timestamps = set() + self.counter_data = dict() + for c in counter_set.flatten().itervalues(): + self.counter_data[c.name] = self.AggregatedCounter(c) + + def _AddMetric(self, metric): + """Adds a single metric sample to the aggregation. Metric samples must be added in + chronological order. + """ + machine = metric.machine_id + time = metric.timestamp + payload = DotDict(json.loads(metric.payload)).flatten() + + self.machines.add(machine) + self.timestamps.add(time) + for k in payload: + if k not in self.counter_data: + continue + val = payload.get(k, None) + if val is not None: + self.counter_data[k].AddSample(machine, time, val) + + @classmethod + def CreateAggregateForTimespan(cls, client, group_key, start_time, end_time, counter_set, callback): + """Creates an Aggregated Metric and for a set of counters and queries the database, + aggregating all metrics for the given cluster and interval across the given time span. + Invokes the given callback with the resulting AggregatedMetric object after the query is + completed. + """ + aggregator = AggregatedMetric(group_key, start_time, end_time, counter_set) + + def _OnQueryPartial(metrics): + if len(metrics) > 0: + Metric.QueryTimespan(client, group_key, start_time, end_time, _OnQueryPartial, + excl_start_key=metrics[-1].GetKey()) + for m in metrics: + aggregator._AddMetric(m) + else: + callback(aggregator) + + Metric.QueryTimespan(client, group_key, start_time, end_time, _OnQueryPartial) diff --git a/backend/db/notification.py b/backend/db/notification.py new file mode 100644 index 0000000..96c6378 --- /dev/null +++ b/backend/db/notification.py @@ -0,0 +1,176 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder notification. + + Notifications are the mechanism by which the client is notified of incremental modifications + to server assets. Notifications enable the client to incrementally keep its state in sync + with the server. Furthermore, pushed alerts notify end users that their client does not + have the latest data. + + *Every* change that is visible to a particular user results in the creation of a notification. + Certain kinds of operations, such as share, also result in the push of an alert to all the + user's devices. Operations which modify viewpoint assets in ways that are visible to other + followers also result in the creation of an activity. + + Although at first glance notifications and activities are similar, they are different in + important ways: + + 1. Notifications are created *per-user*, and apply to any change that may have occurred in + assets viewable by that user. In contrast, activities are associated with viewpoints, + not users, and are only created for changes that are visible to all followers. As an + example, a user might override the title of a viewpoint. A notification is created, but + an activity is not, since that change is user-specific and not visible to other followers. + + 2. Notifications contain coarse-granularity invalidation lists, which instruct the client as + to which assets need to be re-queried. In contrast, activities contain the exhaustive list + of operation-specific asset identifiers which were changed. As an example, a notification + generated by a share might contain just a single episode invalidation, whereas the + corresponding activity would contain the identifier of that episode *plus* the identifiers + of all photos shared as part of that episode. + + 3. The notification table will be truncated periodically, whereas the activity table lives + indefinitely. The notification table exists in order to allow the client to incrementally + update its cache of server state. The activity table exists in order to keep a record of + structural changes to the viewpoint. + + Notification: Notify client of changes to asset tree. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import json +import logging + +from tornado import gen +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + +@DBObject.map_table_attributes +class Notification(DBRangeObject): + """Viewfinder notification data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.NOTIFICATION) + + def __init__(self, user_id=None, notification_id=None): + super(Notification, self).__init__() + self.user_id = user_id + self.notification_id = notification_id + + def GetInvalidate(self): + """Parses and returns the JSON invalidate attribute as a python dict.""" + return json.loads(self.invalidate) if self.invalidate is not None else None + + def SetInvalidate(self, invalidate_dict): + """Sets invalidation python dict as JSON invalidate attribute.""" + self.invalidate = json.dumps(invalidate_dict) + + @classmethod + @gen.coroutine + def TryClearBadge(cls, client, user_id, device_id, notification_id): + """Tries to create a "clear_badges" notification with the given id. Returns False if another + notification with this id has already been created, else returns True. + """ + notification = Notification(user_id, notification_id) + notification.name = 'clear_badges' + notification.timestamp = util.GetCurrentTimestamp() + notification.sender_id = user_id + notification.sender_device_id = device_id + notification.badge = 0 + + # If _TryUpdate returns false, then new notifications showed up while the query was running, and so + # retry creation of the notification. + success = yield notification._TryUpdate(client) + raise gen.Return(success) + + @classmethod + @gen.coroutine + def QueryLast(cls, client, user_id, consistent_read=False): + """Returns the notification with the highest notification_id, or None if the notification + table is empty. + """ + notification_list = yield gen.Task(Notification.RangeQuery, + client, + user_id, + range_desc=None, + limit=1, + col_names=None, + scan_forward=False, + consistent_read=consistent_read) + + raise gen.Return(notification_list[0] if len(notification_list) > 0 else None) + + @classmethod + @gen.coroutine + def CreateForUser(cls, client, operation, user_id, name, invalidate=None, + activity_id=None, viewpoint_id=None, seq_num_pair=None, + inc_badge=False, consistent_read=False): + """Creates a notification database record for the specified user, based upon the + notification record that was last created and the current operation. If "inc_badge" is + true, then increment the user's pending notification badge count. Returns the newly + created notification. + """ + while True: + last_notification = yield Notification.QueryLast(client, user_id, consistent_read=consistent_read) + + if last_notification is None: + notification_id = 1 + badge = 0 + else: + notification_id = last_notification.notification_id + 1 + badge = last_notification.badge + + notification = Notification(user_id, notification_id) + notification.name = name + if invalidate is not None: + notification.SetInvalidate(invalidate) + notification.activity_id = activity_id + notification.viewpoint_id = viewpoint_id + + # Store update_seq and/or viewed_seq on notification if they were specified. + if seq_num_pair is not None: + update_seq, viewed_seq = seq_num_pair + notification.update_seq = update_seq + + # viewed_seq applies only to the user that submitted the operation. + if viewed_seq is not None and operation.user_id == user_id: + notification.viewed_seq = viewed_seq + + # Increment badge if requested to do so. + if inc_badge: + badge += 1 + notification.badge = badge + notification.timestamp = operation.timestamp + notification.sender_id = operation.user_id + notification.sender_device_id = operation.device_id + notification.op_id = operation.operation_id + + success = yield notification._TryUpdate(client) + + # If creation of the notification succeeded, then query is complete. Otherwise, retry from + # start since another notification allocated the same id. + if success: + raise gen.Return(notification) + + # If update failed, may have been because we couldn't read the "real" last notification. + consistent_read = True + + @gen.coroutine + def _TryUpdate(self, client): + """Creates a new notification database record using the next available notification_id. + Avoids race conditions by using the "expected" argument to Update in order to ensure that + a unique notification_id is used. If another notification allocates a particular + notification_id first, this method will return False. The caller can then retry with a new + notification_id. + """ + try: + yield gen.Task(self.Update, client, expected={'notification_id': False}) + except Exception as e: + # Notification creation failed, so return False so caller can retry. + logging.info('notification id %d is already in use: %s' % (self.notification_id, e)) + raise gen.Return(False) + + raise gen.Return(True) diff --git a/backend/db/operation.py b/backend/db/operation.py new file mode 100644 index 0000000..a812dfa --- /dev/null +++ b/backend/db/operation.py @@ -0,0 +1,293 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder operation. + +Operations are write-ahead logs for functions which comprise multiple steps. If server failure +or transient exceptions would otherwise leave the database in a partial state, creating an +operation to encapsulate the execution will guarantee that it is retried until successful. +Operations must be written to be idempotent, so that executing them once or many times results +in the same ending state. + +Operations are submitted to the "OpManager", which contains the functionality necessary to +execute the operations and handle any failures. See the header to op_manager.py for more details. + + Operation: write-ahead log record for mutating request +""" + +__authors__ = ['andy@emailscrubbed.com (Andy Kimball)', + 'spencer@emailscrubbed.com (Spencer Kimball)'] + +import json +import logging +import sys +import time + +from functools import partial +from tornado import gen, stack_context +from tornado.concurrent import return_future +from viewfinder.backend.base import message, util +from viewfinder.backend.base.exceptions import StopOperationError, FailpointError, TooManyRetriesError +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.asset_id import IdPrefix, ConstructAssetId, DeconstructAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.op.op_context import OpContext +from viewfinder.backend.op.op_manager import OpManager + + +@DBObject.map_table_attributes +class Operation(DBRangeObject): + """Viewfinder operation data object.""" + __slots__ = ['log_handler', 'context', '_triggered_failpoints'] + + _table = DBObject._schema.GetTable(vf_schema.OPERATION) + _user_table = DBObject._schema.GetTable(vf_schema.USER) + + ANONYMOUS_DEVICE_ID = 0 + ANONYMOUS_USER_ID = 0 + + FAILPOINTS_ENABLED = False + """If false, then calls to TriggerFailpoint are no-ops.""" + + def __init__(self, user_id=None, operation_id=None): + super(Operation, self).__init__() + self.user_id = user_id + self.operation_id = operation_id + self.context = {} # general purpose context dictionary that any consumers or Operation may use. + self._triggered_failpoints = None + + def IsBackedOff(self): + """Returns true if this operation is in exponential backoff awaiting a retry.""" + return self.backoff > time.time() + + @gen.coroutine + def SetCheckpoint(self, client, checkpoint): + """Stores progress information with the operation. If the operation is restarted, it can + use this information to skip over steps it's already completed. The progress information + is operation-specific and is not used in any way by the operation framework itself. The + checkpoint is expected to be a JSON-serializable dict. + """ + assert Operation.GetCurrent() == self, 'checkpoint should only be set during op execution' + assert isinstance(checkpoint, dict), checkpoint + self.checkpoint = checkpoint + yield self.Update(client) + + @classmethod + def ConstructOperationId(cls, device_id, uniquifier): + """Returns an operation id constructed from component parts. See "ConstructAssetId" for + details of the encoding. + """ + return ConstructAssetId(IdPrefix.Operation, device_id, uniquifier) + + @classmethod + def DeconstructOperationId(cls, operation_id): + """Returns the components of an operation id: device_id, and uniquifier.""" + return DeconstructAssetId(IdPrefix.Operation, operation_id) + + @classmethod + @gen.coroutine + def VerifyOperationId(cls, client, user_id, device_id, operation_id): + """Ensures that a client-provided operation id is valid according to the rules specified + in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Operation, operation_id, has_timestamp=False) + + @classmethod + def GetCurrent(cls): + """Returns the operation currently being executed. If no operation is being executed, + returns a default operation with user_id and device_id set to None. + """ + current = OpContext.current() + if current is not None and current.executing_op is not None: + return current.executing_op + return Operation() + + @classmethod + @gen.coroutine + def CreateNested(cls, client, method, args): + """Creates a new nested operation, which is based on the current operation. The current + operation is stopped so that the nested operation can be run. The nested operation must + complete successfully before the parent operation will be continued. + + The new operation's id parenthesizes the current operation id. For example: + current op_id: o12345 + nested op_id: (o12345) + + This ensures that at most one nested operation runs at a time (and that it sorts and + therefore runs before the current op), and makes it easy to identify nested operations + when debugging. + """ + current = OpContext.current() + assert current is not None and current.executing_op is not None, \ + 'outer operation must be running in order to execute a nested operation' + current_op = current.executing_op + + assert 'headers' not in args, 'headers are derived from the current operation' + args['headers'] = {'op_id': '+%s' % current_op.operation_id, + 'op_timestamp': current_op.timestamp} + + nested_op = yield gen.Task(Operation.CreateAndExecute, + client, + current_op.user_id, + current_op.device_id, + method, + args) + + # If nested op is in quarantine, then fail this operation, since it cannot start until the + # nested op has successfully completed. + if nested_op.quarantine: + raise TooManyRetriesError('Nested operation "%s" already exists and is in quarantine.' % nested_op.operation_id) + + raise StopOperationError() + + @classmethod + @gen.engine + def CreateAndExecute(cls, client, user_id, device_id, method, args, callback, + message_version=message.MAX_SUPPORTED_MESSAGE_VERSION): + """Creates a new operation with 'method' and 'args' describing the operation. After + successfully creating the operation, the operation is asynchronously executed. Returns + the op that was executed. + """ + # Get useful headers and strip all else. + headers = args.pop('headers', {}) + synchronous = headers.pop('synchronous', False) + + # Validate the op_id and op_timestamp fields. + op_id = headers.pop('op_id', None) + op_timestamp = headers.pop('op_timestamp', None) + assert (op_id is not None) == (op_timestamp is not None), (op_id, op_timestamp) + + # Validate that op_id is correctly formed and is allowed to be generated by the current device. + # No need to do this if the op_id was generated by the system as part of message upgrade. + if op_id is not None and headers.get('original_version', 0) >= message.Message.ADD_OP_HEADER_VERSION: + yield Operation.VerifyOperationId(client, user_id, device_id, op_id) + + # Use the op_id provided by the user, or generate a system op-id. + if op_id is None: + op_id = yield gen.Task(Operation.AllocateSystemOperationId, client) + + # Possibly migrate backwards to a message version that is compatible with older versions of the + # server that may still be running. + op_message = message.Message(args, default_version=message.MAX_MESSAGE_VERSION) + yield gen.Task(op_message.Migrate, + client, + migrate_version=message_version, + migrators=OpManager.Instance().op_map[method].migrators) + + op = Operation(user_id, op_id) + op.device_id = device_id + op.method = method + op.json = json.dumps(args) + op.attempts = 0 + + # Set timestamp to header value if it was specified, or current timestamp if not. + if op_timestamp is not None: + op.timestamp = op_timestamp + else: + op.timestamp = util.GetCurrentTimestamp() + + # Set expired backoff so that if this process fails before the op can be executed, in the worst + # case it will eventually get picked up by the OpManager's scan for failed ops. Note that in + # rare cases, this may mean that the op gets picked up immediately by another server (i.e. even + # though the current server has *not* failed), but that is fine -- it doesn't really matter what + # server executes the op, it just matters that the op gets executed in a timely manner. + op.backoff = 0 + + # Try to create the operation if it does not yet exist. + try: + yield gen.Task(op.Update, client, expected={'operation_id': False}) + + # Execute the op according to the 'synchronous' parameter. If 'synchronous' is True, the + # callback is invoked only after the operation has completed. Useful during unittests to + # ensure the mutations wrought by the operation are queryable. + logging.info('PERSIST: user: %d, device: %d, op: %s, method: %s' % (user_id, device_id, op_id, method)) + except Exception: + # Return existing op. + logging.warning('operation "%s" already exists', op_id) + existing_op = yield gen.Task(Operation.Query, client, user_id, op_id, None, must_exist=False) + if existing_op is not None: + op = existing_op + + # If not synchronous, we fire the callback, but continue to execute. + if not synchronous: + callback(op) + + # Establish new "clean" context in which to execute the operation. The operation should not rely + # on any context, since it may end up run on a completely different machine. In addition, establish + # an exception barrier in order to handle any bugs or asserts, rather than letting the context + # established for the request handle it, since it will have already completed). + with stack_context.NullContext(): + with util.ExceptionBarrier(util.LogExceptionCallback): + OpManager.Instance().MaybeExecuteOp(client, user_id, op.operation_id) + else: + # Let exceptions flow up to request context so they'll be put into an error response. + OpManager.Instance().MaybeExecuteOp(client, user_id, op.operation_id, partial(callback, op)) + + @classmethod + def CreateAnonymous(cls, client, method, args, callback): + """Similar to CreateAndExecute(), but uses the anonymous user and device and allocates the + operation id from the id-allocator table. + """ + Operation.CreateAndExecute(client, Operation.ANONYMOUS_USER_ID, + Operation.ANONYMOUS_DEVICE_ID, method, args, callback) + + @classmethod + def WaitForOp(cls, client, user_id, operation_id, callback): + """Waits for the specified operation to complete. WaitForOp behaves exactly like using the + "synchronous" option when submitting an operation. The callback will be invoked once the + operation has completed or if it's backed off due to repeated failure. + """ + OpManager.Instance().MaybeExecuteOp(client, user_id, operation_id, callback) + + @classmethod + def ScanFailed(cls, client, callback, limit=None, excl_start_key=None): + """Scans the Operation table for operations which have failed and for which the backoff + time has expired. These operations can be retried. Returns a tuple containing the failed + operations and the key of the last scanned operation. + """ + now = time.time() + Operation.Scan(client, None, callback, limit=limit, excl_start_key=excl_start_key, + scan_filter={'backoff': db_client.ScanFilter([now], 'LE')}) + + @classmethod + @gen.engine + def AllocateSystemOperationId(cls, client, callback): + """Create a unique operation id that is generated using the system device allocator.""" + device_op_id = yield gen.Task(Device.AllocateSystemObjectId, client) + op_id = Operation.ConstructOperationId(Device.SYSTEM, device_op_id) + callback(op_id) + + @classmethod + @gen.coroutine + def TriggerFailpoint(cls, client): + """Raises a non-abortable exception in order to cause the operation to restart. Only raises + the exception if this failpoint has not yet been triggered for this operation. + + This facility is useful for testing operation idempotency in failure situations. + """ + # Check whether failpoint support is enabled. + if not Operation.FAILPOINTS_ENABLED: + return + + op = Operation.GetCurrent() + assert op.operation_id is not None, \ + 'TriggerFailpoint can only be called in scope of executing operation' + + # Get list of previously triggered failpoints for this operation. + triggered_failpoints = op.triggered_failpoints or [] + + # Check whether this failpoint has already been triggered for this operation. + frame = sys._getframe().f_back + trigger_point = [frame.f_code.co_filename, frame.f_lineno] + if trigger_point in triggered_failpoints: + return + + # This is first time the failpoint has been triggered, so trigger it now and save it to the op. + triggered_failpoints.append(trigger_point) + op = Operation.CreateFromKeywords(user_id=op.user_id, + operation_id=op.operation_id, + triggered_failpoints=list(triggered_failpoints)) + yield gen.Task(op.Update, client) + + raise FailpointError(*trigger_point) diff --git a/backend/db/photo.py b/backend/db/photo.py new file mode 100644 index 0000000..d21aee4 --- /dev/null +++ b/backend/db/photo.py @@ -0,0 +1,207 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder photo. + + Photo: viewfinder photo information +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging + +from tornado import gen, httpclient, options +from viewfinder.backend.base import constants +from viewfinder.backend.base.exceptions import InvalidRequestError, PermissionError, ServiceUnavailableError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.asset_id import IdPrefix, ConstructTimestampAssetId, DeconstructTimestampAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.user_photo import UserPhoto +from viewfinder.backend.resources.message.error_messages import SERVICE_UNAVAILABLE + + +@DBObject.map_table_attributes +class Photo(DBHashObject): + """Viewfinder photo data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.PHOTO) + + # The period (in seconds) for which a photo that was shared directly + # with other users can be fully unshared. + CLAWBACK_GRACE_PERIOD = constants.SECONDS_PER_WEEK + + # There's a set of attributes which are allowed to be updated when creating + # photo metadata when the photo metadata already exists (but not after S3 upload of data). This is to handle a + # special case in the client where MD5 generation for photo data isn't deterministic (platform weirdness). + PHOTO_CREATE_ATTRIBUTE_UPDATE_ALLOWED_SET = ('tn_md5', 'med_md5', 'orig_md5', 'full_md5') + + def __init__(self, photo_id=None): + super(Photo, self).__init__() + self.photo_id = photo_id + + def MakeMetadataDict(self, post, user_post, user_photo): + """Constructs a dictionary containing photo metadata attributes, overridden by post and + user post attributes where required. + """ + ph_dict = self._asdict() + labels = post.labels.combine() + if user_post is not None: + labels = labels.union(user_post.labels.combine()) + asset_keys = set() + if user_photo is not None and user_photo.asset_keys: + asset_keys.update(user_photo.asset_keys) + if asset_keys: + ph_dict['asset_keys'] = list(UserPhoto.MakeAssetFingerprintSet(asset_keys)) + if len(labels) > 0: + ph_dict['labels'] = list(labels) + ph_dict.pop('client_data', None) + return ph_dict + + @classmethod + def ConstructPhotoId(cls, timestamp, device_id, uniquifier): + """Returns a photo id constructed from component parts. Photos + sort from newest to oldest. See "ConstructTimestampAssetId" for + details of the encoding. + """ + return ConstructTimestampAssetId(IdPrefix.Photo, timestamp, device_id, uniquifier) + + @classmethod + def DeconstructPhotoId(cls, photo_id): + """Returns the components of a photo id: timestamp, device_id, and + uniquifier. + """ + return DeconstructTimestampAssetId(IdPrefix.Photo, photo_id) + + @classmethod + @gen.coroutine + def VerifyPhotoId(cls, client, user_id, device_id, photo_id): + """Ensures that a client-provided photo id is valid according + to the rules specified in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Photo, photo_id, has_timestamp=True) + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **ph_dict): + """Creates a new photo metadata object from the provided dictionary. + + Returns: new photo. + """ + assert 'photo_id' in ph_dict and 'user_id' in ph_dict and 'episode_id' in ph_dict, ph_dict + photo = Photo.CreateFromKeywords(**ph_dict) + yield photo.Update(client) + raise gen.Return(photo) + + @classmethod + @gen.coroutine + def UpdateExisting(cls, client, **ph_dict): + """Updates existing photo metadata from the provided dictionary.""" + assert 'timestamp' not in ph_dict and 'episode_id' not in ph_dict and 'user_id' not in ph_dict, ph_dict + photo = Photo.CreateFromKeywords(**ph_dict) + yield photo.Update(client) + + @classmethod + @gen.coroutine + def CheckCreate(cls, client, **ph_dict): + """For a photo that already exists, check that its attributes match. + Return: photo, if it already exists. + """ + assert 'photo_id' in ph_dict and 'user_id' in ph_dict, ph_dict + photo = yield Photo.Query(client, ph_dict['photo_id'], None, must_exist=False) + # All attributes should match between the ph_dict and persisted photo metadata + # (except those allowed to be different). + if photo is not None and \ + photo.HasMismatchedValues(Photo.PHOTO_CREATE_ATTRIBUTE_UPDATE_ALLOWED_SET, **ph_dict): + logging.warning('Photo.CheckCreate: keyword mismatch failure: %s, %s' % (photo, ph_dict)) + raise InvalidRequestError('There is a mismatch between request and persisted photo metadata during photo ' + 'metadata creation.') + raise gen.Return(photo) + + @classmethod + @gen.coroutine + def IsImageUploaded(cls, obj_store, photo_id, suffix): + """Determines whether a photo's image data has been uploaded to S3 + by using a HEAD request. If the image exists, then invokes callback + with the Etag of the image. Otherwise, invokes the callback with None. + """ + url = obj_store.GenerateUrl(photo_id + suffix, method='HEAD') + http_client = httpclient.AsyncHTTPClient() + try: + response = yield http_client.fetch(url, + method='HEAD', + validate_cert=options.options.validate_cert) + except httpclient.HTTPError as e: + if e.code == 404: + raise gen.Return(None) + else: + logging.warning('Photo store S3 HEAD request error: [%s] %s' % (type(e).__name__, e.message)) + raise ServiceUnavailableError(SERVICE_UNAVAILABLE) + + if response.code == 200: + raise gen.Return(response.headers['Etag']) + else: + raise AssertionError('failure on HEAD request to photo %s: %s' % + (photo_id + suffix, response)) + + @classmethod + @gen.coroutine + def UpdatePhoto(cls, client, act_dict, **ph_dict): + """Updates photo to metadata object from the provided dictionary.""" + assert 'photo_id' in ph_dict and 'user_id' in ph_dict, ph_dict + photo = yield Photo._CheckUpdate(client, **ph_dict) + + assert photo is not None, ph_dict + assert photo.photo_id == ph_dict['photo_id'], (photo, ph_dict) + assert photo.user_id == ph_dict['user_id'], (photo, ph_dict) + + asset_keys = ph_dict.pop('asset_keys', None) + if asset_keys: + assert ph_dict['user_id'] + + up_dict = {'user_id': ph_dict['user_id'], + 'photo_id': ph_dict['photo_id'], + 'asset_keys': asset_keys} + user_photo = UserPhoto.CreateFromKeywords(**up_dict) + else: + user_photo = None + + photo.UpdateFromKeywords(**ph_dict) + + # Aborts are NOT allowed after this point because we're about to modify db state. + # Ensure that we haven't modified it yet. + client.CheckDBNotModified() + + yield photo.Update(client) + + if user_photo is not None: + yield user_photo.Update(client) + + @classmethod + @gen.coroutine + def _CheckUpdate(cls, client, **ph_dict): + """Checks that the photo exists. Checks photo metadata object against the provided dictionary. + Checks that the user_id in the dictionary matches the one on the photo. + Returns: photo + """ + assert 'photo_id' in ph_dict and 'user_id' in ph_dict, ph_dict + photo = yield Photo.Query(client, ph_dict['photo_id'], None, must_exist=False) + + if photo is None: + raise InvalidRequestError('Photo "%s" does not exist and so cannot be updated.' % + ph_dict['photo_id']) + + if photo.user_id != ph_dict['user_id']: + raise PermissionError('User id of photo does not match requesting user') + + raise gen.Return(photo) + + @classmethod + @gen.coroutine + def UpdateOperation(cls, client, act_dict, ph_dict): + """Updates photo metadata.""" + assert ph_dict['user_id'] == Operation.GetCurrent().user_id + + # Call helper to carry out update of the photo. + yield Photo.UpdatePhoto(client, act_dict=act_dict, **ph_dict) diff --git a/backend/db/post.py b/backend/db/post.py new file mode 100644 index 0000000..c3092a8 --- /dev/null +++ b/backend/db/post.py @@ -0,0 +1,92 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Post relation. + +A post is the relationship between a photo and an episode. The post +table allows quick queries for all photos within an episode. + + Post: defines relation between a photo and an episode +""" + +__author__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import logging + +from tornado import gen +from functools import partial +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.asset_id import IdPrefix +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.db.user_post import UserPost + +@DBObject.map_table_attributes +class Post(DBRangeObject): + """Post data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.POST) + + UNSHARED = 'unshared' + REMOVED = 'removed' + + def IsUnshared(self): + """Returns true if the photo has been unshared by the posting user, + making it inaccessible to followers of the viewpoint. + """ + return Post.UNSHARED in self.labels + + def IsRemoved(self): + """Returns true if the photo has been removed by the posting user, making it inaccessible + to followers of the viewpoint. This label will always be set if the unshared label is set. + """ + # TODO(Andy): Get rid of the UNSHARED check once every unshared post is also removed. + return Post.UNSHARED in self.labels or Post.REMOVED in self.labels + + @classmethod + def ConstructPostId(cls, episode_id, photo_id): + """Returns a post id constructed by concatenating the id of the + episode that contains the post with the id of the posted photo. + The two parts are separated by a '+' character, which is not produced + by the base64hex encoding, and so can be used to later deconstruct + the post id if necessary. While the key for a post is composite with + hash_key=episode_id and range_key=photo_id, the range key for the + UserPost table must use a concatenation of the post key as its range + key. ConstructPostId, and its inverse DeconstructPostId, concatenate + and split the post key respectively into a single value. This + concatenated post id will sort posts in the same order that + (episode_id, photo_id) does. This is because each part is encoded + using base64hex, which sorts in the same way as the source value. + Furthermore, the '+' character sorts lower than any of the base64hex + bytes, so the episode_id terminates with a "low" byte, making substring + episode ids sort lower. + """ + return IdPrefix.Post + episode_id[1:] + '+' + photo_id[1:] + + @classmethod + def DeconstructPostId(cls, post_id): + """Returns the components of a post id: (episode_id, photo_id).""" + assert post_id[0] == IdPrefix.Post, post_id + index = post_id.index('+') + assert index > 0, post_id + + return IdPrefix.Episode + post_id[1:index], IdPrefix.Photo + post_id[index + 1:] + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **post_dict): + """Creates a new post from post_dict. The caller is responsible for checking permission to + do this, as well as ensuring that the post does not yet exist (or is just being identically + rewritten). + + Posts are sorted using the photo id. Since the photo id is prefixed by the photo timestamp, + this amounts to sorting by photo timestamp, which is in order from newest to oldest (i.e. + descending). + + Returns: post that was created. + """ + post = Post.CreateFromKeywords(**post_dict) + yield gen.Task(post.Update, client) + raise gen.Return(post) diff --git a/backend/db/query_parser.py b/backend/db/query_parser.py new file mode 100644 index 0000000..d4bbd40 --- /dev/null +++ b/backend/db/query_parser.py @@ -0,0 +1,620 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Parses query expressions into hierarchical query trees. + +Given a data schema (defined via schema.py), you can create Query objects +and execute them against a database client. For example, with table 'User' +and columns 'given_name' and 'Email', you can run a query as follows: + + query_str = '(user.given_name=spencer | user.given_name="andrew") & user.family_name=kimball' + users = User.Query(query_str).Evaluate(client) + + + Query: query object to evaluate query expression. + +Supports parameterized phrases which should be used with strings from an untrusted source in order to mitigate +injection attacks. The keys are valid in the phrase part of the query where the key name is surrounded by braces. +Parameterized queries are passed as a tuple where the first element is the query string followed by a dictionary +with the parameter keys mapped to parameter values. The above query expression using parameters: + + bound_query_str = ('(user.given_name={p1} | user.given_name={p2}) & user.family_name={p3}', + {'p1': 'spencer', 'p2': 'andrew', 'p3': 'kimball'}) + +The parameter keys may be made up of letters, digits, and the underscore (_). + +Based on calculator.py by Andrew Brehaut & Steven Ashley of picoparse. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import re + +from bisect import bisect_left, bisect_right +from collections import namedtuple +from functools import partial +from picoparse import one_of, choice, many1, tri, commit, p, many_until1 +from picoparse.text import run_text_parser, as_string, lexeme, whitespace, quoted +from string import digits, letters +from tornado import escape +from viewfinder.backend.base import util +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.schema import IndexedTable + +_query_cache = util.LRUCache(100) + +# Optimization for the common case: Indexer.GetQueryString usually returns a single token +# as a quoted string. Picoparse's character-by-character operation is slow, so bypass +# it when we can. +_QUOTED_TOKEN_RE = re.compile(r'^"([^"\\]*)"$') + +class _MatchResult(object): + """The result of a successful match after node evaluation. Data + initially contains the value attached to a key in a posting list, + but may be arbitrarily updated and augmented as it moves through the + query tree. + """ + __slots__ = ['key', 'data'] + + def __init__(self, key, data): + self.key = key + self.data = data + + def __cmp__(self, other): + return cmp(self.key, other.key) + + +# The result from evaluating a node or subtree of the query tree. +# Matches is a list of '_MatchResult' values. 'last_key' is the +# last key in the range which was successfully evaluated. +_EvalResult = namedtuple('_EvalResult', ['matches', 'last_key', 'read_units']) + + +class _QueryNode(object): + """Query nodes form binary trees with set operations at each + intermediate node and ranges of keys at leaf nodes. + """ + pass + + +class _OpNode(_QueryNode): + """Operation nodes implement set operations on the results of + key range queries. Set operations include union, difference and + intersection. + """ + def __init__(self, schema, left): + """Takes the left child as a parameter. The right child is merged + into this operation via a call to Merge. The two are followed + recursively during evaluation, with the left and right evaluated + asynchronously. When both have completed, the result of the set + operation (union, difference, intersection & positional intersection) + is evaluated and returned to parent. + """ + self._left = left + self._right = None + + def PrintTree(self, level, param_dict): + """Depth-first printout of tree structure for debugging.""" + return self._OpName() + ' __ ' + self._left.PrintTree(level + 1, param_dict) + '\n' + level * (' ') + \ + ' \_ ' + self._right.PrintTree(level + 1, param_dict) + + def Merge(self, right): + """Rearranges the parent-child relationship to conform to operator + precedence. If the precedence of the current node is >= the node + to its right, then the right-hand node becomes the new parent, + with the current node made its new left-hand node (meaning this + node will be evaluated first, as the evaluation is a depth-first + traversal). Otherwise, the right-hand node simply stays the right + hand node. + """ + if self._precedence >= right._precedence: + self._right = right._left + right._left = self + return right + else: + self._right = right + return self + + def Evaluate(self, client, callback, start_key, consistent_read, param_dict): + """Recursively evaluates the query tree via a depth- first + traversal. Returns the result set, defined by the data delivered + via the IndexTermNodes and then operated on by the OpNodes. + """ + with util.ArrayBarrier(partial(self._SetOperation, callback)) as b: + self._left.Evaluate(client, b.Callback(), start_key, consistent_read, param_dict) + self._right.Evaluate(client, b.Callback(), start_key, consistent_read, param_dict) + + def _OpName(self): + raise NotImplementedError() + + def _SetOperation(self): + raise NotImplementedError() + + +class Union(_OpNode): + """Returns the union of two sets. Lowest precedence.""" + def __init__(self, schema, left): + super(Union, self).__init__(schema, left) + self._precedence = 0 + + def _OpName(self): + return "| " + + def _SetOperation(self, callback, results): + """For union, the sets are additively combined. Both the first and + last keys are defined as the minimum of first and last keys for + the two results. This jibes well with the intuitive notion that we + were able to successfully evaluate the union between these two + sets for all values starting at the very minimum up to and + including the minimum of the two last keys. This prevents us from + over-stepping a gap between the two ranges for example. + """ + assert len(results) == 2 + matches = results[0].matches + results[1].matches + matches.sort() + + if results[0].last_key is None: + last_key = results[1].last_key + elif results[1].last_key is None: + last_key = results[0].last_key + else: + last_key = min(results[0].last_key, results[1].last_key) + + # Only returns matches up to last_key. + if last_key: + matches = matches[:bisect_right(matches, _MatchResult(last_key, None))] + callback(_EvalResult(matches=matches, last_key=last_key, + read_units=results[0].read_units + results[1].read_units)) + + +class Difference(_OpNode): + """Returns the difference of two sets.""" + def __init__(self, schema, left): + super(Difference, self).__init__(schema, left) + self._precedence = 1 + + def _OpName(self): + return "- " + + def _SetOperation(self, callback, results): + """For set difference, the ordering matters. The second set can be + thought of as a mask over the first set. The overlap starting at + the first key and extending to the min(last_key1, last_key2) defines + the successfully-evaluated range. To see this, consider that any + portion of the subtracted range which lies before the first set is + irrelevant (as we can only eval what we have of the first range + going forward), and any portion of the subtracted range which lies + after the first set can't yet be evaluated, as we don't yet know + the remainder of the first set. + """ + assert len(results) == 2 + m1 = results[0].matches + m2 = results[1].matches + matches = [] + while m1: + m2_idx = bisect_left(m2, m1[0]) + if m2_idx == len(m2): + break + elif m1[0] == m2[m2_idx]: + # A match, so skip. + m1 = m1[1:] + elif m2_idx == 0: + # No match, so include the range of m1's up to the start of m2. + m1_idx = bisect_left(m1, m2[0]) + if m1_idx == len(m1): + # The last result of m1 is still before the next m2; include all. + matches += m1 + break + else: + # Include up to the next possible match between m1 and m2. + matches += m1[:m1_idx] + m1 = m1[m1_idx:] + + if results[0].last_key is None: + last_key = results[1].last_key + elif results[1].last_key is None: + last_key = results[0].last_key + else: + last_key = min(results[0].last_key, results[1].last_key) + callback(_EvalResult(matches=matches, last_key=last_key, + read_units=results[0].read_units + results[1].read_units)) + + +class Intersection(_OpNode): + """Returns the intersection of two sets.""" + def __init__(self, schema, left): + super(Intersection, self).__init__(schema, left) + self._precedence = 2 + + def _OpName(self): + return "& " + + def _SetOperation(self, callback, results): + """Efficiently skip through the two lists using list bisection. + """ + assert len(results) == 2 + m1 = results[0].matches + m2 = results[1].matches + matches = [] + while m1 and m2: + if m1[0] < m2[0]: + m1 = m1[bisect_left(m1, m2[0]):] + elif m2[0] < m1[0]: + m2 = m2[bisect_left(m2, m1[0]):] + else: + matches.append(m1[0]) + m1 = m1[1:] + m2 = m2[1:] + + callback(_EvalResult(matches=matches, last_key=self._ComputeLastKey(results), + read_units=results[0].read_units + results[1].read_units)) + + def _ComputeLastKey(self, results): + """For set intersection, the successfully-evaluated portion starts + at the maximum of the first keys and extends as far as the minimum + of the last keys. This is the portion for which we have enough + information to definitively determine intersection. + """ + if results[0].last_key is None: + last_key = results[1].last_key + elif results[1].last_key is None: + last_key = results[0].last_key + else: + last_key = min(results[0].last_key, results[1].last_key) + return last_key + + +class PositionalIntersection(Intersection): + """Returns the intersection of two posting lists, but only return + matches where relative positions between the left and right nodes + are self._delta apart. + """ + def __init__(self, schema, left): + super(PositionalIntersection, self).__init__(schema, left) + self._delta = 1 + + def Merge(self, right): + """Merging for positional intersections works a little differently + than for normal operations. Here, we are on the lookout to merge + place-holder (None) nodes out of the tree. If the right-hand node + we're trying to merge is a place-holder, then return the left-hand + node as the result of the merge (this removes trailing + place-holders). If the right-hand node we're merging's left-hand + node is a place-holder, we skip over the right-hand node by + attaching its right-hand node in its place. This operation + increases our delta by the delta of the now-merged right-hand node. + """ + if right is None: + return self._left + elif isinstance(right, PositionalIntersection) and right._left is None: + self._delta += right._delta + self._right = right._right + else: + self._right = right + return self + + def _OpName(self): + return "+%d" % self._delta + + def _SetOperation(self, callback, results): + """Perform a standard intersection operation, but on a match, + return only the left node's position data, and then only those + positions with a difference of self._delta from the right node's + positions. + """ + assert len(results) == 2 + m1 = results[0].matches + m2 = results[1].matches + matches = [] + while m1 and m2: + if m1[0] < m2[0]: + m1 = m1[bisect_left(m1, m2[0]):] + elif m2[0] < m1[0]: + m2 = m2[bisect_left(m2, m1[0]):] + else: + new_data = [pos for pos in m1[0].data if (pos + self._delta in m2[0].data)] + if new_data: + matches.append(_MatchResult(key=m1[0].key, data=new_data)) + m1 = m1[1:] + m2 = m2[1:] + + callback(_EvalResult(matches=matches, last_key=self._ComputeLastKey(results), + read_units=results[0].read_units + results[1].read_units)) + + +class Parenthetical(_QueryNode): + """This node encapsulates a child node and will be merged into + _OpNodes as if it were a single value; This protects parenthesized + trees from having their order adjusted. + """ + def __init__(self, schema, child): + # Collapse any nodes which have a missing left-hand node (this + # can happen with a top-level positional intersection node that + # has a place-holder term in the left-most position). + if isinstance(child, PositionalIntersection) and child._left is None: + child = child._right + + self._child = child + self._precedence = 1000 + + def PrintTree(self, level, param_dict): + return self._child.PrintTree(level, param_dict) + + def Evaluate(self, client, callback, start_key, consistent_read, param_dict): + self._child.Evaluate(client, callback, start_key, consistent_read, param_dict) + + +class PhraseNode(_QueryNode): + """This node encapsulates a phrase, which may be a single term, or + an arbitrarily complex subtree. For example, a phrase query with + term expansions. The first pass through the parser merely sets the + phrase on creation. However, when the tree is initialized, the + phrase subtree is created. + """ + def __init__(self, schema, table_name, column_name, phrase): + """Convert the phrase into a query string using the appropriate + indexer object (found via table:column in schema). This query + string is passed to the phrase parser, and the new query subtree + is set as the child of this PhraseNode. + """ + self._precedence = 1000 + + try: + self.schema = schema + self.table = self.schema.GetTable(table_name) + assert isinstance(self.table, IndexedTable), table_name + column = self.table.GetColumn(column_name) + assert column.indexer, column_name + self.column = column + self.phrase = phrase + self.child_parser = _PhraseParser(self.schema, self.table, column) + except: + logging.exception('phrase \'%s\'' % phrase) + raise + + def _CreateChildNode(self, param_dict): + if isinstance(self.phrase, Parameter): + phrase = self.phrase.Resolve(param_dict) + else: + assert isinstance(self.phrase, basestring) + phrase = self.phrase + phrase_query_str = self.column.indexer.GetQueryString(self.column, phrase) + match = _QUOTED_TOKEN_RE.match(phrase_query_str) + if match is not None: + # Fast path: it's one token, so create the IndexTermNode directly. + return IndexTermNode(self.schema, self.table, self.column, match.group(1)) + else: + # Slow path: run the real query parser to make sure we tokenize more complex queries correctly. + return self.child_parser.Run(phrase_query_str) + + def PrintTree(self, level, param_dict): + child = self._CreateChildNode(param_dict) + return child.PrintTree(level, param_dict) + + def Evaluate(self, client, callback, start_key, consistent_read, param_dict): + child = self._CreateChildNode(param_dict) + child.Evaluate(client, callback, start_key, consistent_read, param_dict) + + +class IndexTermNode(_QueryNode): + """Accesses the posting list for the indexed term from table:column. + The term itself is indexed according to the rules of the column + indexer and the posting lists for each emitted term are queried. + """ + def __init__(self, schema, table, column, index_term): + self._table = table + self._column = column + self._index_term = escape.utf8(index_term) + self._precedence = 1000 + self._start_key = None + self._last_key = None + self._matches = None + + def PrintTree(self, level, param_dict): + return self._index_term + + def Evaluate(self, client, callback, start_key, consistent_read, param_dict): + """Queries the database for keys beginning with start_key, with a + limit defined in the table schema. Consistent reads are disabled + as they're unlikely to make a difference in search results (and + are half as expensive in the DynamoDB cost model). + """ + def _OnQuery(result): + self._start_key = start_key + self._last_key = result.last_key.range_key if result.last_key is not None else None + self._matches = [_MatchResult( + key=item['k'], data=self._Unpack(item.get('d', None))) for item in result.items] + callback(_EvalResult(matches=self._matches, last_key=self._last_key, + read_units=result.read_units)) + + if self._start_key and self._start_key <= start_key and \ + self._last_key and self._last_key > start_key: + self._start_key = start_key + self._matches = self._matches[ + bisect_right(self._matches, _MatchResult(key=start_key, data=None)):] + callback(_EvalResult(matches=self._matches, last_key=self._last_key, + read_units=0)) + else: + excl_start_key = db_client.DBKey(self._index_term, start_key) if start_key is not None else None + client.Query(table=vf_schema.INDEX, hash_key=self._index_term, + range_operator=None, attributes=None, callback=_OnQuery, + limit=vf_schema.SCHEMA.GetTable(vf_schema.INDEX).scan_limit, + consistent_read=consistent_read, excl_start_key=excl_start_key) + + def _Unpack(self, data): + return self._column.indexer.UnpackFreight(self._column, data) + + +class Parameter(object): + def __init__(self, param_name): + self.param_name = param_name + + def __str__(self): + return '{%s}' % self.param_name + + def Resolve(self, param_dict): + param_value = None if param_dict is None else param_dict.get(self.param_name, None) + assert param_value is not None, 'No value for query parameter: %s' % self.param_name + return param_value + + +class _QueryParser(object): + def __init__(self, schema, parse_phrase_func=None): + self._schema = schema + + if not parse_phrase_func: + parse_phrase_func = self._ParsePhrase + + self._op_classes = {'|': Union, '-': Difference, + '&': Intersection, '+': PositionalIntersection} + self._operator = p(lexeme, p(one_of, u''.join(self._op_classes.keys()))) + token_char = p(one_of, letters + digits + ':_') + self._token = as_string(p(many1, token_char)) + self._phrase = p(choice, quoted, self._token) + self._param_parser = p(choice, self._ParseParam, quoted, self._token) + self._term_parser = p(choice, self._ParseParenthetical, parse_phrase_func) + self._expr_parser = p(choice, self._ParseOp, self._term_parser) + + def Run(self, query_str): + """Runs the parser on the provided query expression and returns + the resulting query tree. + """ + # Convert query_str to Unicode since picoparser expects Unicode for non-ASCII characters. + query_tree, _ = run_text_parser(self._expr_parser, escape.to_unicode(query_str)) + return query_tree + + @tri + def _ParseOp(self): + """Consumes one operation, defined by left term and right + expression, which may be either another term or another ParseOp(). + """ + left = self._term_parser() + op = self._operator() + commit() + right = self._expr_parser() + whitespace() + node = self._op_classes[op](self._schema, left) + return node.Merge(right) + + @tri + def _ParseParenthetical(self): + """Consumes parenthetical expression.""" + whitespace() + one_of('(') + commit() + whitespace() + node = self._expr_parser() + whitespace() + one_of(')') + whitespace() + return Parenthetical(self._schema, node) + + @tri + def _ParseParam(self): + """Consumes parameter keys for lookup in a parameter dictionary passed in with the query. + Parameter keys may consist of letters, digits and underscores (_). + Returns the value that the parameter key maps to. + """ + one_of('{') + param_name = ''.join(many_until1(p(one_of, letters + digits + '_'), p(one_of, '}'))[0]) + return Parameter(param_name) + + @tri + def _ParsePhrase(self): + """Consumes a key range specification of the form .=. + """ + whitespace() + table = self._token().lower() + one_of('.') + commit() + column = self._token().lower() + whitespace() + one_of('=') + whitespace() + phrase = self._param_parser() + node = PhraseNode(self._schema, table, column, phrase) + whitespace() + return node + + +class _PhraseParser(_QueryParser): + def __init__(self, schema, table, column): + super(_PhraseParser, self).__init__(schema, self._ParseIndexTerm) + self._table = table + self._column = column + + def _ParseIndexTerm(self): + """Consumes an index term. If '_', creates a place-holder node + (None); otherwise, creates an IndexTermNode. + """ + whitespace() + index_term = self._phrase() + if index_term == '_': + node = None + else: + node = IndexTermNode( + self._schema, self._table, self._column, index_term) + whitespace() + return node + + +class Query(object): + """Parses a query into a hierarchical tree of set operations + in the context of the provided data schema. + """ + def __init__(self, schema, query_str): + """Parses query_str into a query node tree.""" + self._query_str = query_str + self._query_tree = _QueryParser(schema).Run(query_str) + + def PrintTree(self, param_dict): + print self._query_str, param_dict + print self._query_tree.PrintTree(0, param_dict) + + def Evaluate(self, client, callback, limit=50, start_key=None, end_key=None, + consistent_read=False, param_dict=None): + """Evaluates the query tree according to the provided db + client. 'callback' is invoked with the query results. + + Returns keys matching the query expression, up to the limit. + """ + def _OnEvaluate(results, read_units, eval_result): + results += [mr.key for mr in eval_result.matches if (not end_key or mr.key < end_key)] + read_units += eval_result.read_units + reached_end_key = end_key is not None and eval_result.last_key >= end_key + reached_limit = limit is not None and len(results) >= limit + if eval_result.last_key is None or reached_end_key or reached_limit: + logging.debug('query required %d read units' % read_units) + results = results[:limit] + callback(results) + else: + logging.debug('query under limit at key %s (%d < %s), ' + '%d read units; extending query' % + (repr(eval_result.last_key), len(results), + limit if limit is not None else 'all', read_units)) + self._query_tree.Evaluate(client, partial(_OnEvaluate, results, read_units), + eval_result.last_key, consistent_read, param_dict) + + results = [] + self._query_tree.Evaluate(client, partial(_OnEvaluate, results, 0), + start_key, consistent_read, param_dict) + + +def CompileQuery(schema, bound_query_str): + """Returns a bound query (a (Query, param_dict) pair) for the given schema and query expression. + + bound_query_str can be either a query string, or a (query_str, param_dict) pair. + (This interface is used for consistency with an older interface). + + Usage: + query, param_dict = query_parser.CompileQuery(schema, (query_str, param_dict)) + query.Evaluate(..., param_dict) + """ + if isinstance(bound_query_str, tuple): + query_str, param_dict = bound_query_str + else: + query_str = bound_query_str + param_dict = None + query = _query_cache.Get((schema, query_str), lambda: Query(schema, query_str)) + return query, param_dict diff --git a/backend/db/range_base.py b/backend/db/range_base.py new file mode 100644 index 0000000..522b454 --- /dev/null +++ b/backend/db/range_base.py @@ -0,0 +1,178 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Base object for database objects from tables with a composite key +{hash-key, range-key}. + + DBRangeObject: superclass of all composite-key data objects +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from functools import partial +from tornado import gen +from viewfinder.backend.base import util +from viewfinder.backend.db import db_client, schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.versions import Version + +class DBRangeObject(DBObject): + """Base class for items from tables built with a composite + {hash-key, range-key} key. + """ + __slots__ = [] + + def __init__(self, columns=None): + super(DBRangeObject, self).__init__(columns=columns) + + @classmethod + def Query(cls, client, hash_key, range_key, col_names, callback, + must_exist=True, consistent_read=False): + """Queries a object by composite hash/range key.""" + cls.KeyQuery(client, key=db_client.DBKey(hash_key=hash_key, range_key=range_key), + col_names=col_names, callback=callback, must_exist=must_exist, + consistent_read=consistent_read) + + @classmethod + def Allocate(cls, client, hash_key, callback): + """Allocates a new range key via the id_allocator table. + Instantiates a new object using provided 'hash_key' and the + allocated 'range_key' and Invokes the provided callback with new ojb. + """ + assert cls._allocator, 'class has no id allocator declared' + def _OnAllocate(range_key): + o = cls(hash_key, range_key) + o._columns[schema.Table.VERSION_COLUMN.name].Set(Version.GetCurrentVersion()) + callback(o) + cls._allocator.NextId(client, _OnAllocate) + + @classmethod + @gen.engine + def RangeQuery(cls, client, hash_key, range_desc, limit, col_names, callback, + excl_start_key=None, consistent_read=False, count=False, scan_forward=True): + """Executes a range query using the predicate contained in 'range_desc' + to select a set of items. If 'limit' is not None, then the database will + be queried until 'limit' items have been fetched, or until there are no + more items to fetch. If 'limit' is None, then the first page of results + is returned (i.e. whatever DynamoDB returns). + + 'range_desc' is a tuple of ([range_key], ('EQ'|'LE'|'LT'|'GE'|'GT'|'BEGINS_WITH')), + --or-- ([range_start_key, range_end_key], 'BETWEEN'). + + If 'excl_start_key' is not of type DBKey, assumes that 'excl_start_key' + only specifies the range key and build an appropriate DBKey object using + hash_key to feed to the db client interface. + + On completion, invokes callback with a list of queried objects. If + count is True, invokes callback with count. + """ + if limit == 0: + assert not count + callback([]) + return + + if not count: + col_set = cls._CreateColumnSet(col_names) + attrs = [cls._table.GetColumn(name).key for name in col_set] + else: + attrs = None + + if excl_start_key is not None and not isinstance(excl_start_key, db_client.DBKey): + excl_start_key = db_client.DBKey(hash_key, excl_start_key) + + instance_count = 0 + instances = [] + while True: + remaining = limit - instance_count if limit is not None else None + query_result = yield gen.Task(client.Query, table=cls._table.name, hash_key=hash_key, + range_operator=range_desc, attributes=attrs, + limit=remaining, consistent_read=consistent_read, + count=count, excl_start_key=excl_start_key, scan_forward=scan_forward) + + instance_count += query_result.count + if not count: + for item in query_result.items: + instance = cls._CreateFromQuery(**item) + instances.append(instance) + + assert limit is None or instance_count <= limit, (limit, instance_count) + if query_result.last_key is None or limit is None or instance_count == limit: + callback(instance_count if count else instances) + break + + excl_start_key = query_result.last_key + + @classmethod + def VisitRange(cls, client, hash_key, range_desc, col_names, visitor, callback, + consistent_read=False, scan_forward=True): + """Query for all items in the specified key range. For each key, + invoke the "visitor" function: + + visitor(object, visit_callback) + + When the visitor function has completed the visit, it should invoke + "visit_callback" with no parameters. Once all object keys have been + visited, then "callback" is invoked. + """ + def _OnQuery(items): + if len(items) < DBObject._VISIT_LIMIT: + barrier_callback = callback + else: + barrier_callback = partial(_DoQuery, excl_start_key=items[-1].GetKey()) + + with util.Barrier(barrier_callback) as b: + for item in items: + visitor(item, callback=b.Callback()) + + def _DoQuery(excl_start_key): + cls.RangeQuery(client, hash_key, range_desc, limit=DBObject._VISIT_LIMIT, + col_names=col_names, excl_start_key=excl_start_key, + callback=_OnQuery, consistent_read=consistent_read) + + _DoQuery(None) + + def GetKey(self): + """Returns the object's composite (hash, range) key.""" + return db_client.DBKey( + hash_key=self._columns[self._table.hash_key_col.name].Get(), + range_key=self._columns[self._table.range_key_col.name].Get()) + + @classmethod + def _MakeIndexKey(cls, db_key): + """Creates an indexing key from the provided object key. This is an + amalgamation of the composite key. Separates the hash and range + keys by a colon ':'. This method is symmetric with _ParseIndexKey. + + Override for more efficient formulation (e.g. Breadcrumb). + """ + hash_key = util.ConvertToString(db_key.hash_key) + range_key = util.ConvertToString(db_key.range_key) + index_key = '%d:' % len(hash_key) + hash_key + range_key + return index_key + + @classmethod + def _ParseIndexKey(cls, index_key): + """Returns a tuple representing the object's composite key by + parsing the provided index key. This is symmetric with + _MakeIndexKey, and is used to extract the actual object key from + results of index queries. + """ + colon_loc = index_key.find(':') + assert colon_loc != -1, index_key + hash_key_len = int(index_key[:colon_loc]) + index_key = index_key[colon_loc + 1:] + hash_key = index_key[:hash_key_len] + range_key = index_key[hash_key_len:] + if cls._table.hash_key_col.value_type == 'N': + hash_key = int(hash_key) + if cls._table.range_key_col.value_type == 'N': + range_key = int(range_key) + return db_client.DBKey(hash_key=hash_key, range_key=range_key) + + @classmethod + def _GetIndexedObjectClass(cls): + """Can be overridden by derived range-type classes to specify what + class of object can be created from a parsed index key, if not the + DBRangeObject-derived class itself. For example, Breadcrumb index + keys yield User instances, but Post index keys yield Post instances. + """ + return cls diff --git a/backend/db/schema.py b/backend/db/schema.py new file mode 100644 index 0000000..210f9fa --- /dev/null +++ b/backend/db/schema.py @@ -0,0 +1,997 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Schema definition. + +The first set of classes in this file handle the underlying storage +for a single column of a datastore object. The base class for this, +_Value, defines the interface. There are three subclasses: +_SingleValue, which handles one piece of data, whatever the type +(e.g. an integer, floating point, latitude/longitude pair, string, +etc.); _SetValue, which handles a set of _SingleValue +data; and _KeyValue, which hold an immutable key value. + +_SetValue objects are used to represent one or more similar +items for an object. For example, a user object might contain one +email string for each verified identity. + +This module provides the following utilities for packing / unpacking +data between a database-friendly ASCII data and structured named tuples. + + PackLocation: Location from named tuple to DB data + UnpackLocation: from DB data to Location named tuple + PackPlacmark: Placemark from named tuple to DB data + UnpackPlacemark: from DB data to Placemark named tuple + +This module provides the following external classes: + + Column: a single value column definition + SetColumn: a set of values column definition + Table: Contains one or more columns + IndexedTable: Variant of Table which maintains secondary indexes + IndexTable: Variant of Table which stores index info + Schema: Contains one or more tables + SchemaException: exception class for schema operations +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import json +import logging +import struct + +from collections import namedtuple +from functools import partial +from tornado import options +from viewfinder.backend.base import base64hex, secrets, util +from viewfinder.backend.db import db_client, indexers + +options.define('delete_vestigial', default=False, help='deletes vestigial tables') +options.define('verify_provisioning', default=True, + help='abort if provisioned capacity does not match schema values') + +Location = namedtuple('Location', ['latitude', 'longitude', 'accuracy']) + +Placemark = namedtuple('Placemark', ['iso_country_code', 'country', 'state', 'locality', + 'sublocality', 'thoroughfare', 'subthoroughfare']) + + +def PackLocation(location): + """Converts 'location' named tuple into a packed, base64-hex-encoded + string representation for storage in DynamoDB. + """ + packed = struct.pack('>ddd', *[float(x) for x in location]) + return base64hex.B64HexEncode(packed) + +def UnpackLocation(value): + """Converts from a packed, base64-hex-encoded representation of + latitude, longitude and accuracy into a Location namedtuple. + """ + packed = base64hex.B64HexDecode(value) + latitude, longitude, accuracy = struct.unpack('>ddd', packed) + return Location(latitude=latitude, longitude=longitude, accuracy=accuracy) + + +def PackPlacemark(placemark): + """Converts 'placemark' named tuple into a packed, + base64-hex-encoded, comma-separated representation for storage in + DynamoDB. + """ + return ','.join([base64hex.B64HexEncode(x.encode('utf-8'), padding=False) for x in placemark]) + +def UnpackPlacemark(value): + """Converts from a comma-separated, base64-hex-encoded + representation of hierarchical place names into 'Placemark' named + tuple with an utf-8 encoded place names. + """ + pm_values = [] + for x in value.split(','): + try: + # TODO(spencer): the rstrip() below is necessary as data in the + # index has already been encoded with a bug in the base64 padding + # We need to rebuild the index before reverting this. + decoded_x = base64hex.B64HexDecode(x.rstrip('='), padding=False).decode('utf-8') + except: + decoded_x = '' + pm_values.append(decoded_x) + return Placemark(*pm_values) + + +class SchemaException(Exception): + pass + + +class _Value(object): + """Holds a column value. Each column type decides how to store its + in-memory representation. Some column types are like _PlacemarkValue, + where the raw db value is stored in memory as a more useful Python + object. Other column types are like _JSONValue -- the raw db value + is stored in memory and converted to a useful Python object on demand. + """ + __slots__ = ['col_def', '_modified', '_value'] + + def __init__(self, col_def): + self.col_def = col_def + self._modified = False + self._value = None + + def IsModified(self): + return self._modified + + def SetModified(self, modified): + self._modified = modified + + def Get(self, asdict=False): + """Returns the value of the column in a format that is convenient + for use in Python. If 'asdict' is true, then convert to a Python + dict if this column type supports it. + """ + raise NotImplementedError() + + def Load(self, value): + """Loads the value of the column from the format that is stored in the + database. Sets the IsModified bit to false. + """ + raise NotImplementedError() + + def Set(self, value): + """Updates the value of the column with any type that can be converted + into the column type. Sets the IsModified bit to true if the value of + the column actually changes. + """ + raise NotImplementedError() + + def Del(self): + raise NotImplementedError() + + def Update(self): + raise NotImplementedError() + + def OnUpdate(self): + """Called on completion of an update.""" + self.SetModified(False) + + def IndexTerms(self): + """Returns an index term dict in conjunction with PUT. If the term + dict is empty, returns an empty term dict. In either case, the + previous set of index terms is queried and the difference between + the old and new sets is used to delete or add this object's key to + the term posting lists. + """ + assert self.col_def.indexer + try: + if self.Get(): + index_terms = self.col_def.indexer.Index(self.col_def, self.Get()) + else: + index_terms = {} + except: + logging.exception('generation of index terms for %s' % repr(self.Get())) + index_terms = {} + + return db_client.UpdateAttr(value=index_terms or None, action='PUT') + + def _CheckType(self, value): + """Ensure that "value" has a type that matches the type of the column. + """ + def _CheckSingleValue(single_value): + assert single_value != '', value + if self.col_def.value_type in ['S', 'SS']: + is_expected_type = type(single_value) in [str, unicode] + else: + assert self.col_def.value_type in ['N', 'NS'], self.col_def + is_expected_type = type(single_value) in [int, long, float] + + assert is_expected_type, \ + (self.col_def.table.name, self.col_def.value_type, self.col_def.name, value) + + if value is not None: + if self.col_def.value_type in ['SS', 'NS']: + [_CheckSingleValue(v) for v in value if v is not None] + else: + _CheckSingleValue(value) + + +class _SingleValue(_Value): + """Holds a column with a single value (such as a string, a timestamp, etc.). + """ + def Get(self, asdict=False): + """Returns the value in the raw db format by default.""" + return self._value + + def Load(self, value): + """Stores the raw db value by default.""" + self._CheckType(value) + self._value = value + + def Set(self, value): + """Stores the raw db value by default.""" + assert value != '', 'DynamoDB does not support setting attributes to the empty string' + if value != self._value: + assert not self.col_def.read_only or self._value is None, \ + 'cannot modify read-only column "%s": %s=>%s' % (self.col_def.name, self._value, repr(value)) + self._CheckType(value) + self.SetModified(True) + self._value = value + + def Update(self): + """Returns PUT and the new value if modified. If new value is + None, returns DELETE. + """ + assert self.IsModified() + if self._value is not None: + return db_client.UpdateAttr(value=self._value, action='PUT') + else: + return db_client.UpdateAttr(value=None, action='DELETE') + + +class _LatLngValue(_Value): + """Subclass of _Value that holds latitude, longitude and an accuracy + measure, all double-precision floating point values. They are + stored together as a base64hex-encoded struct-packed string for + storage, but are available as a Location namedtuple. + + This value may be set via either Set(Location(latitude, longitude, accuracy)), + Set({'latitude': , 'longitude': , 'accuracy': }), + or Set(packed-b64hex-encoded-string). + """ + def Get(self, asdict=False): + """Gets the value as a Location or dict object.""" + if asdict: + return self._value._asdict() + else: + return self._value + + def Load(self, value): + """Stores the raw db str as a Location object in memory.""" + assert isinstance(value, (str, unicode)), value + self._value = UnpackLocation(value) + + def Set(self, value): + """Converts 'value' to a Location and store.""" + if value is None: + location = None + elif isinstance(value, dict): + location = Location(**value) + elif isinstance(value, (str, unicode)): + location = UnpackLocation(value) + else: + assert isinstance(value, Location), value + location = value + + if location != self._value: + self.SetModified(True) + self._value = location + + def Update(self): + """Returns PUT and the new value if modified. If new value is + None, returns DELETE. + """ + assert self.IsModified() + if self._value is not None: + return db_client.UpdateAttr(value=PackLocation(self._value), action='PUT') + else: + return db_client.UpdateAttr(value=None, action='DELETE') + + +class _PlacemarkValue(_Value): + """Subclass of _Value that holds hierarchical placenames. They are stored + together in the datastore as a url-encoded, comma-separated string, but are + available in python as a Placemark namedtuple. + + This value may be set via either Set(Placemark), or + Set({'iso_country_code', 'country': , 'state': , + 'locality': , 'sublocality': , + 'thoroughfare': , 'subthoroughfare': + }), or Set(). + """ + def Get(self, asdict=False): + """Gets the value as a Placemark or dict object.""" + if asdict: + # Cannot return empty strings for missing placemark fields as + # JSON validator doesn't allow empty strings. + return dict([(k, v) for k, v in self._value._asdict().items() \ + if v is not None and v != '']) + else: + return self._value + + def Load(self, value): + """Stores the raw db str as a Placemark object in memory.""" + assert isinstance(value, (str, unicode)), value + self._value = UnpackPlacemark(value) + + def Set(self, value): + """Converts 'value' to a Placemark and store.""" + if value is None: + placemark = None + elif isinstance(value, dict): + placemark = Placemark(value.get('iso_country_code', ''), value.get('country', ''), + value.get('state', ''), value.get('locality', ''), + value.get('sublocality', ''), value.get('thoroughfare', ''), + value.get('subthoroughfare', '')) + elif isinstance(value, (str, unicode)): + placemark = UnpackPlacemark(value) + else: + assert isinstance(value, Placemark), value + placemark = value + + if placemark != self._value: + self.SetModified(True) + self._value = placemark + + def Update(self): + """Returns PUT and the new value if modified. If new value is + None, returns DELETE. + """ + assert self.IsModified() + if self._value is not None: + return db_client.UpdateAttr(value=PackPlacemark(self._value), action='PUT') + else: + return db_client.UpdateAttr(value=None, action='DELETE') + + +class _JSONValue(_Value): + """Subclass of _Value that holds a python data structure, which is + stored as a JSON-encoded string. + """ + def Get(self, asdict=False): + """Returns the JSON-encoded string converted to a Python data type.""" + if self._value: + return json.loads(self._value) + else: + return None + + def Load(self, value): + """Stores the raw string value loaded from the db.""" + assert isinstance(value, (str, unicode)), value + self._value = value + + def Set(self, value): + """Converts 'value' to a JSON-encoded string before storing it.""" + value = util.ToCanonicalJSON(value) + if value != self._value: + self.SetModified(True) + self._value = value + + def Update(self): + assert self.IsModified() + if self._value is not None: + return db_client.UpdateAttr(value=self._value, action='PUT') + else: + return db_client.UpdateAttr(value=None, action='DELETE') + + +class _DelayedCrypt(object): + """This class delays the decryption of an encrypted value until the class is invoked. This + level of indirection ensures that the caller must take explicit action in order to decrypt + a value. This helps to prevent accidental logging or use of the plaintext. + """ + def __init__(self, encrypted_value): + self._encrypted_value = encrypted_value + + def Decrypt(self): + """Returns the decrypted value.""" + crypter = _CryptValue._GetCrypter() + return json.loads(crypter.Decrypt(self._encrypted_value)) + + def __eq__(self, other): + """Returns true if self._encrypted_value is equal to the other's _encrypted_value.""" + if isinstance(other, _DelayedCrypt): + return self._encrypted_value == other._encrypted_value + return NotImplemented + + def __ne__(self, other): + """Returns true if self._encrypted_value is not equal to the other's _encrypted_value.""" + if isinstance(other, _DelayedCrypt): + return self._encrypted_value != other._encrypted_value + return NotImplemented + + +class _CryptValue(_JSONValue): + """Subclass of _Value that holds a python data structure, which is stored as a JSON-encoded + string that has been encrypted with the service-wide db crypt key. + """ + @classmethod + def _GetCrypter(cls): + if not hasattr(cls, '_crypter'): + cls._crypter = secrets.GetCrypter('db_crypt') + return cls._crypter + + def Get(self, asdict=False): + """Gets the encrypted value as an instance of _DelayedCrypt. This instance's Decrypt method + must be invoked in order to extract the unencrypted value. See the docs for _DelayedCrypt + for details. + """ + if self._value is not None: + # Return JSON-friendly format when _asdict() is used. The default JSONEncoder does not + # handle anything other than the basic Python types. Note that in this case, decrypting + # the value via invocation is not possible, but it is useful for getting an object as a + # dict, and then serializing or copying it elsewhere. + if asdict: + return {'__crypt__': self._value} + + return _DelayedCrypt(self._value) + else: + return None + + def Set(self, value): + """Converts 'value' to a JSON-encoded string and encrypts it before storing it.""" + if value is None: + encrypted_value = None + elif isinstance(value, _DelayedCrypt): + encrypted_value = value._encrypted_value + elif isinstance(value, dict) and '__crypt__' in value: + encrypted_value = value['__crypt__'] + else: + crypter = _CryptValue._GetCrypter() + encrypted_value = crypter.Encrypt(json.dumps(value)) + + if self._value != encrypted_value: + self.SetModified(True) + self._value = encrypted_value + + +class _LayeredSet(frozenset): + """A special frozenset subclass which handles deletions and + additions without necessarily knowing about the contents of the + canonical set (as it exists at the current moment in the + datastore). This is useful for describing changes to a set of + values in the datastore without requiring the contents be + queried. It redefines add, clear, discard & remove by augmenting + additional, internal set() objects to track additions and + deletions. These are then used to update the datastore as + incremental changes. + + Once a value has been removed, it cannot then be added and vice- + versa. Values which have been added or removed are transient and + have no effect on tests for whether an element is 'in' the set, on + set equality, or on set operations. These types of set operations + are valid only with the results of set values queried from the + datastore. clear() adds all elements which were queried (ones in the + underlying frozenset) to the deleted set. Any elements previously + added to additions are discarded. + + The return values of various methods must be judiciously + interpreted. Asking if 'x in _LayeredSet' may yield an answer only + in relation to imperfect knowledge of the canonical set. + """ + def __init__(self, s=[]): + super(_LayeredSet, self).__init__(s) + self.additions = set() + self.deletions = set() + + def __repr__(self): + return '%s +%s -%s' % (super(_LayeredSet, self).__repr__(), + self.additions.__repr__(), + self.deletions.__repr__()) + + def add(self, elem): + assert not self.deletions + assert elem != '' + if elem not in self: + self.additions.add(elem) + + def clear(self): + self.deletions = set(self) + self.additions.clear() + + def discard(self, elem): + self.remove(elem) + + def remove(self, elem): + assert not self.additions + self.deletions.add(elem) + + def combine(self): + """Return a set that adds the additions and removes the deletions.""" + return self.additions.union(self).difference(self.deletions) + +class _SetValue(_Value): + """Holds a set of values using a LayeredSet to keep track of + incremental additions and deletions. + """ + def __init__(self, col_def): + super(_SetValue, self).__init__(col_def) + self._value = _LayeredSet() + + def IsModified(self): + """True if _modified or if additions or deletions are not empty.""" + return self._modified or self._value.additions or self._value.deletions + + def Get(self, asdict=False): + """Returns the partial set.""" + if asdict: + return list(self._value) + else: + return self._value + + def Load(self, value): + """Stores the raw set value as a LayeredSet.""" + assert value is None or isinstance(value, (list, tuple, set, frozenset)), type(value) + if value is None: + self._value = _LayeredSet() + else: + self._value = _LayeredSet(value) + + def Set(self, value): + """Sets the contents of the entire set. This sets a flag which + indicates that the DynamoDB update should use a PUT action to + replace the previous contents of the set. + """ + self.SetModified(True) + self.Load(value) + + def Update(self): + """Returns an action {ADD, DELETE, PUT} and the set of values for + an update depending on the state of the layered set. If the set + was assigned directly, use PUT. If there are set additions, use + ADD; otherwise DELETE. + """ + if self._modified: + if self._value.additions: + value = list(self._value.union(self._value.additions)) + else: + value = list(self._value.difference(self._value.deletions)) + + # DynamoDB does not support PUT of empty set, so instead DELETE the attribute entirely. + if not value: + return db_client.UpdateAttr(None, action='DELETE') + + return db_client.UpdateAttr(value=value, action='PUT') + else: + if self._value.additions: + return db_client.UpdateAttr(value=list(self._value.additions), action='ADD') + elif self._value.deletions: + return db_client.UpdateAttr(value=list(self._value.deletions), action='DELETE') + else: + assert False, 'Update called with unmodified set' + + def OnUpdate(self): + """Called on completion of an update.""" + self._modified = False + new_set = self._value.combine() + self._value = _LayeredSet(new_set) + + def IndexTerms(self): + """Returns the set of index terms in conjunction with the action, + which is one of {PUT, ADD, DELETE}. Index terms which are meant to + replace the former set are returned with PUT. This requires the + previous terms be queried. The differences between the old and the + new term sets determines which old terms are deleted and which new + terms are added to the index. ADD and DELETE do not require the + previous terms be queried. + """ + assert self.col_def.indexer and \ + isinstance(self.col_def.indexer, indexers.SecondaryIndexer) + update = self.Update() + # Create the term dict. + term_dict = {} + if update.value is not None: + for term in update.value: + term_dict.update(self.col_def.indexer.Index(self.col_def, term).items()) + return db_client.UpdateAttr(value=term_dict, action=update.action) + + +class _KeyValue(_SingleValue): + """Holds a column with a single value (such as a string, a timestamp, etc.). + """ + def Set(self, value): + if self._value is None and value is not None: + self._CheckType(value) + self.SetModified(True) + self._value = value + elif self._value != value: + assert False, "cannot modify a key value: %s=>%s" % (self._value, repr(value)) + + def Update(self): + """Key values are not updated. On creation, the key is already + specified as part of the request. + """ + assert self.IsModified() + return None + + +class Column(object): + """A single-value column. + + The '_type' values are specified as 'struct' format characters: + http://docs.python.org/library/struct.html + """ + def __init__(self, name, key, value_type, indexer=None, read_only=False): + self.name = name.lower() + self.key = key.lower() + self.value_type = value_type + self.read_only = read_only + self.indexer = indexer + # The back link to the table is set by the containing table. + self.table = None + + def NewInstance(self): + return _SingleValue(self) + + +class HashKeyColumn(Column): + """A column to designate the primary key of a row in the datastore. + In DynamoDB, this is referred to as the 'hash-key', and is used to + randomly & uniformly disperse items in a particular table across the + key range. + """ + def __init__(self, name, key, value_type): + super(HashKeyColumn, self).__init__(name, key, value_type, indexer=None) + + def NewInstance(self): + return _KeyValue(self) + + +class RangeKeyColumn(Column): + """A column to designate the secondary key of a row in the datastore. + In DynamoDB, this is referred to as the 'range-key', and is used to + provide a sort order on items with identical 'hash-key' values. + """ + def __init__(self, name, key, value_type, indexer=None): + super(RangeKeyColumn, self).__init__(name, key, value_type, indexer=indexer) + + def NewInstance(self): + return _KeyValue(self) + + +class SetColumn(Column): + """A subclass of column whose column value in the datastore is a + set of values, each with value as specified by value_type. + """ + def __init__(self, name, key, value_type, indexer=None, read_only=False): + super(SetColumn, self).__init__(name, key, value_type, indexer=indexer, read_only=read_only) + + def NewInstance(self): + return _SetValue(self) + + +class IndexTermsColumn(Column): + """A subclass of column for the list of index terms generated by an + indexed column. These columns are special in that they don't actually + create an instance of a value class to hold the contents. They are + ephemeral and exist only in the database; they cannot be accessed via + the column name on a DBObject, as for all other column types. + + The value type is always a string set 'SS'. + """ + def __init__(self, name, key): + super(IndexTermsColumn, self).__init__(name, key, 'SS', indexer=False) + + def NewInstance(self): + raise TypeError('IndexTermsColumn is ephemeral') + + +class LatLngColumn(Column): + """Column subclass to handle geographic coordinates measured in + degrees of latitude and longitude. An accuracy is also include, + measured in meters. Stores values as double precision floating point + numbers via struct. The results are base64hex encoded for storage in + the backend datastore. Takes either a LocationIndexer or + BreadcrumbIndexer depending on the type of geo search desired. + """ + def __init__(self, name, key, indexer=None): + """Creates a geographic location indexer if 'indexed'.""" + if indexer is not None: + assert isinstance(indexer, indexers.BreadcrumbIndexer) or \ + isinstance(indexer, indexers.LocationIndexer) + super(LatLngColumn, self).__init__(name, key, 'S', indexer=indexer) + + def NewInstance(self): + return _LatLngValue(self) + + +class PlacemarkColumn(Column): + """Column to handle hiearchical place names from country to street- + level. Stores value as comma-separated url-quoted string value in + datastore, but makes value available via a namedtuple. If indexer + is not None, must be of type PlacemarkIndexer. + """ + def __init__(self, name, key, indexer=None): + if indexer is not None: + assert isinstance(indexer, indexers.PlacemarkIndexer) + super(PlacemarkColumn, self).__init__(name, key, 'S', indexer) + + def NewInstance(self): + return _PlacemarkValue(self) + + +class JSONColumn(Column): + """Column to handle JSON-encoded python data structure. + """ + def __init__(self, name, key, read_only=False): + super(JSONColumn, self).__init__(name, key, 'S', indexer=None, read_only=read_only) + + def NewInstance(self): + return _JSONValue(self) + + +class CryptColumn(Column): + """Column with contents that are encrypted with the service-wide db + crypt key. + """ + def __init__(self, name, key): + super(CryptColumn, self).__init__(name, key, 'S', None) + + def NewInstance(self): + return _CryptValue(self) + + +class Table(object): + """A table contains an array of Column objects.""" + VERSION_COLUMN = Column('_version', '_ve', 'N') + + def __init__(self, name, key, read_units, write_units, columns, name_in_db=None): + # Add special column for _version, used in migrating the data model + # as new features demand. + columns.append(Table.VERSION_COLUMN) + # Set up back links in each column definition to this table. The columns + # need the back link for the table key when they generate index terms. + for c in columns: + c.table = self + self.name = name + self.name_in_db = name_in_db if name_in_db else name + self.key = key + self._VerifyColumns(columns) + self._all_column_names = [c.name for c in columns] + self._column_names = [c.name for c in columns if not isinstance(c, IndexTermsColumn)] + self._columns = dict([(c.name, c) for c in columns]) + self._key_to_name = dict([(c.key, c.name) for c in columns]) + self.read_units = read_units + self.write_units = write_units + self.hash_key_col = columns[0] + self.hash_key_schema = db_client.DBKeySchema( + name=self.hash_key_col.key, value_type=self.hash_key_col.value_type) + if len(columns) > 1 and isinstance(columns[1], RangeKeyColumn): + self.range_key_col = columns[1] + self.range_key_schema = db_client.DBKeySchema( + name=self.range_key_col.key, value_type=self.range_key_col.value_type) + else: + self.range_key_col = None + self.range_key_schema = None + + def GetColumnName(self, key): + """Returns the column name for a column key.""" + return self._key_to_name[key] + + def GetColumnNames(self, all_columns=False): + """Returns a list of column names (sorted in original order). Specify + 'all_columns' as True to include index term columns as well. + """ + if all_columns: + return self._all_column_names + else: + return self._column_names + + def GetColumns(self, all_columns=False): + """Returns a list of column definitions. Specify 'all_columns' as True + to include index term columns as well. + """ + if all_columns: + return self._columns.values() + else: + return [c for c in self._columns.values() if not isinstance(c, IndexTermsColumn)] + + def GetColumn(self, name): + """Returns the named column definition. Column names are not case + sensitive. + """ + return self._columns[name.lower()] + + def GetColumnByKey(self, key): + """Returns the column definition by key. + """ + return self._columns[self._key_to_name[key]] + + def _VerifyColumns(self, columns): + """Verifies the columns are appropriately configured. + + - First column is a HashKeyColumn + - Only second column may be a RangeKeyColumn + - All column names are unique + - All column keys are unique + - If any columns are indexed, table is IndexedTable + - SetColumns may only use SecondaryIndexer + """ + # Verify only one ID column, the first. + assert isinstance(columns[0], HashKeyColumn) + column_keys = set([columns[0].key]) + column_names = set([columns[0].name]) + for i in xrange(1, len(columns)): + c = columns[i] + assert not isinstance(c, HashKeyColumn) + if i >= 2: + assert not isinstance(c, RangeKeyColumn) + assert c.name not in column_names, (c.name, column_names) + column_names.add(c.name) + assert c.key not in column_keys, c.key + column_keys.add(c.key) + if c.indexer: + assert isinstance(self, IndexedTable) + if isinstance(c, SetColumn): + assert isinstance(c.indexer, indexers.SecondaryIndexer) + + +class IndexedTable(Table): + """A table whose data is indexed. An indexed table may not use a + composite key. Each column can specify an optional indexing function + ('indexer' to each column definition). The indexer transforms the + column value into a set of terms, each of which is inserted into an + index table, which has a composite key of hash-key=term, + range-key=obj_key. The data is for the column is optional, but might + include term positions in the document, for example, to support + phrase searches. + + Using an indexed table, you can create a full-text search over table + data (e.g. the captions of all images), or an arbitrary secondary + index (e.g., an ordinal popularity ranking of photos). + + When an indexer generates index terms for a column, the terms are + stored near the column data for subsequent reference. For example, + if the column data are modified, the old terms are compared against + the new terms. Terms which have been discarded (diff between old and + new) are deleted from the index table. Terms which have been added + (diff between new and old) are added to the index table. This also + solves the problem of how to handle changes in the indexers, which + might make it impossible to re-derive the previous set of index + terms in order to delete them. + + For each indexed column we generate an additional column to hold the + list of indexed terms. These are ephemeral and not accessible via + the normal DBObject getters and setters. + """ + def __init__(self, name, key, read_units, write_units, columns, name_in_db=None): + index_term_cols = [IndexTermsColumn(c.name + ':t', c.key + ':t') for c in columns if c.indexer] + columns += index_term_cols + super(IndexedTable, self).__init__(name, key, read_units, write_units, columns, name_in_db=name_in_db) + + +class IndexTable(Table): + """A table used to store reverse index data. This is a composite key + table with the indexed term as the hash key and the doc-id as the + range key. Depending on the application, the doc-id may be an + amalgamation of object key and some other value to affect the order + in which results are fed to the query evaluator. Most commonly, the + doc-id is prefixed with a 'reversed' timestamp to yield doc-ids from + posting lists in order of most to least recent. The type (whether a + string or a number) are specified for both the term and the doc-id + to the constructor. + """ + def __init__(self, name, term_type, key_type, read_units, write_units, scan_limit=50, name_in_db=None): + super(IndexTable, self).__init__(name, 'ix', read_units, write_units, + [HashKeyColumn('term', 't', term_type), + RangeKeyColumn('key', 'k', key_type), + Column('data', 'd', 'S')], name_in_db=name_in_db) + self.scan_limit = scan_limit + + +class Schema(object): + """A collection of table definitions. Table names are not case sensitive.""" + def __init__(self, tables): + """A schema based on the provided sequence of table definitions.""" + # Create dictionary mapping from table name to table instance. Due to upgrades, some tables + # may have a different name in the database, so create a set of those names to be used + # during verification. + self._tables = dict() + self._tables_in_db = dict() + for table in tables: + self.AddTable(table) + + def GetTables(self): + """Returns a list of tables in the schema.""" + return sorted(self._tables.values()) + + def GetTable(self, table): + """Returns the descriptor for the named table.""" + return self._tables[table.lower()] + + def TranslateNameInDb(self, name_in_db): + """Given the name of a table in the database, translate to the name + for that table that the application uses (which may be different if + we've done an upgrade). If the table exists in the database, but not + in the application, just return the name in the database. + """ + key = name_in_db.lower() + return self._tables_in_db[key].name if key in self._tables_in_db else name_in_db + + def AddTable(self, table): + """Adds the specified table to the schema.""" + assert table.name not in self._tables, table + assert table.name_in_db not in self._tables_in_db, table + self._tables[table.name.lower()] = table + self._tables_in_db[table.name_in_db.lower()] = table + + def VerifyOrCreate(self, client, callback, verify_only=False): + """Verifies the schema if it exists or creates it if not. + Verification checks existing tables match the schema definition, + warns of vestigial tables, and creates any tables which are + missing. + + Vestigial tables may be deleted by specifying the --delete_vestigial + command line flag. + + On completion, invokes callback with a list of verified table schemas. + """ + def _OnDescribeTable(table, verify_cb, result): + """Verifies the table description in schema matches the + table in the database. + """ + if verify_only: + verify_cb((table.name, result)) + return + + assert isinstance(result, db_client.DescribeTableResult), result + if options.options.verify_provisioning: + # TODO(mike): Longer term, consider using values read from dymamodb for provisioned throughput as authoritative + # or consider some other mechanism for monitoring mismatches. We shouldn't prevent server startup just + # because of a mismatch in provisioned read or write units. + if table.read_units != result.schema.read_units: + logging.warning('%s: read units mismatch %d != %d', table.name, table.read_units, result.schema.read_units) + if table.write_units != result.schema.write_units: + logging.warning('%s: write units mismatch %d != %d', table.name, table.write_units, result.schema.write_units) + assert table.hash_key_schema == result.schema.hash_key_schema, \ + '%s: hash key schema mismatch %r != %r' % \ + (table.name, table.hash_key_schema, result.schema.hash_key_schema) + assert table.range_key_schema == result.schema.range_key_schema, \ + '%s: range key schema mismatch %r != %r' % \ + (table.name, table.range_key_schema, result.schema.range_key_schema) + assert result.schema.status in ['CREATING', 'ACTIVE'], \ + '%s: table status invalid: %s' % (table.name, result.schema.status) + if result.schema.status == 'CREATING': + logging.info('table %s still in CREATING state...waiting 1s' % table.name) + client.AddTimeout(1.0, partial(_VerifyTable, table, verify_cb)) + return + else: + logging.debug('verified table %s' % table.name) + verify_cb((table.name, result)) + + def _VerifyTable(table, verify_cb): + """Gets table description and verifies via _OnDescribeTable.""" + client.DescribeTable(table=table.name, + callback=partial(_OnDescribeTable, table, verify_cb)) + + def _OnCreateTable(table, verify_cb, result): + """Invoked on creation of a table; moves to verification step.""" + assert isinstance(result, db_client.CreateTableResult), result + logging.debug('created table %s: %s' % (table.name, repr(result.schema))) + _VerifyTable(table, verify_cb) + + def _OnListTables(result): + """First callback with results of a list-tables command. + Creates a results barrier which will collect all table schemas + and return 'callback' on successful verification of all tables. + """ + # Create and/or verifies all tables in schema. + with util.ArrayBarrier(callback) as b: + read_capacity = 0 + write_capacity = 0 + for table in self._tables.values(): + read_capacity += table.read_units + write_capacity += table.write_units + if table.name not in result.tables: + if verify_only: + b.Callback()((table.name, None)) + else: + logging.debug('creating table %s...' % table.name) + client.CreateTable(table=table.name, hash_key_schema=table.hash_key_schema, + range_key_schema=table.range_key_schema, + read_units=table.read_units, write_units=table.write_units, + callback=partial(_OnCreateTable, table, b.Callback())) + else: + _VerifyTable(table, b.Callback()) + + # Warn of vestigial tables. + for table in result.tables: + if table.lower() not in self._tables: + logging.warning('vestigial table %s exists in DB, not in schema' % table) + if options.options.delete_vestigial and options.options.localdb: + logging.warning('deleting vestigial table %s') + client.DeleteTable(table=table, callback=util.NoCallback) + + # Cost metric. + def _CostPerMonth(units, read=True): + return 30 * 24 * 0.01 * (units / (50 if read else 10)) + + logging.debug('total tables: %d' % len(self._tables)) + logging.debug('total read capacity: %d, $%.2f/month' % (read_capacity, _CostPerMonth(read_capacity, True))) + logging.debug('total write capacity: %d, $%.2f/month' % (write_capacity, _CostPerMonth(write_capacity, False))) + + client.ListTables(callback=_OnListTables) diff --git a/backend/db/settings.py b/backend/db/settings.py new file mode 100644 index 0000000..1851c76 --- /dev/null +++ b/backend/db/settings.py @@ -0,0 +1,158 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder settings. + +These classes support the storage of any kind of user or application settings in a single +shared Settings table. For example, user settings can be stored alongside device settings +and internal application settings. + +In order to add a new kind of settings, derive from the Settings class. Override the +_COLUMN_NAMES class attribute in order to specify which columns in the Settings table are +used by the new class. The base Settings class will only expose those columns, in order +to catch bugs where an attempt is made to access a column for the wrong settings group. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class Settings(DBRangeObject): + """Viewfinder settings data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.SETTINGS) + + # List of attribute names in the Settings table that can be read and written by this class. By + # default, the base class loads all settings attributes, but a derived class can override this + # field in order to specify a subset that it manages. + _COLUMN_NAMES = None + + def __init__(self): + col_names = self.__class__._COLUMN_NAMES + ['settings_id', 'group_name', '_version'] + columns = [self._table.GetColumn(name) for name in col_names] if col_names is not None else None + super(Settings, self).__init__(columns=columns) + + +@DBObject.map_table_attributes +class AccountSettings(Settings): + """Settings group that contains options and choices affecting a user account. + + Alert settings: + - email_alerts: enum controlling when alert emails are sent; by default, no alert emails + are sent: + none: do not send alert emails. + on_share_new: send email when new conversation is started or joined. + + - sms_alerts: enum controlling when alert SMS messages are sent; by default, no alert SMS + messages are sent: + none: do not send alert SMS messages. + on_share_new: send SMS message when new conversation is started or joined. + + - push_alerts: enum controlling when notifications are pushed to the user's device(s); by + default no alerts are pushed: + none: do not push any alerts + all: push alerts on share_new, add_followers, share_existing, post_comment + + Storage settings: + - storage_options: set of boolean options controlling photo storage: + use_cloud: cloud storage is enabled for the user account. + store_originals: originals are uploaded to the cloud. + + Marketing communication: + - marketing: enum controlling when marketing communication is sent (default=all): + none: do not send marketing communications + all: send all marketing communications + """ + GROUP_NAME = 'account' + + # Email alerts. + EMAIL_ALERTS = 'email_alerts' + EMAIL_NONE = 'none' + EMAIL_ON_SHARE_NEW = 'on_share_new' + ALL_EMAIL_ALERTS = [EMAIL_NONE, EMAIL_ON_SHARE_NEW] + + # SMS alerts. + SMS_ALERTS = 'sms_alerts' + SMS_NONE = 'none' + SMS_ON_SHARE_NEW = 'on_share_new' + ALL_SMS_ALERTS = [SMS_NONE, SMS_ON_SHARE_NEW] + + # Push alerts. + PUSH_ALERTS = 'push_alerts' + PUSH_NONE = 'none' + PUSH_ALL = 'all' + ALL_PUSH_ALERTS = [PUSH_NONE, PUSH_ALL] + + # Storage options. + STORAGE_OPTIONS = 'storage_options' + USE_CLOUD = 'use_cloud' + STORE_ORIGINALS = 'store_originals' + ALL_STORAGE_OPTIONS = [USE_CLOUD, STORE_ORIGINALS] + + # Marketing communication. + MARKETING = 'marketing' + MARKETING_NONE = 'none' + MARKETING_ALL = 'all' + ALL_MARKETING = [MARKETING_NONE, MARKETING_ALL] + + _COLUMN_NAMES = ['user_id', EMAIL_ALERTS, SMS_ALERTS, PUSH_ALERTS, STORAGE_OPTIONS, MARKETING, 'sms_count'] + """Names of columns that are accessible on this object.""" + + _JSON_ATTRIBUTES = [EMAIL_ALERTS, SMS_ALERTS, STORAGE_OPTIONS] + """Subset of attributes that are returned to the owning user in query_users.""" + + def AllowMarketing(self): + """Returns true if marketing communication is allowed to be sent.""" + return self.marketing != AccountSettings.MARKETING_NONE + + def MakeMetadataDict(self): + """Project a subset of account settings attributes that can be provided to the user.""" + settings_dict = {} + for attr_name in AccountSettings._JSON_ATTRIBUTES: + value = getattr(self, attr_name, None) + if isinstance(value, frozenset): + util.SetIfNotEmpty(settings_dict, attr_name, list(value)) + else: + util.SetIfNotNone(settings_dict, attr_name, value) + + return settings_dict + + @classmethod + def ConstructSettingsId(cls, user_id): + """Constructs the settings id used for user settings: us:.""" + return 'us:%d' % user_id + + @classmethod + def ConstructKey(cls, user_id): + """Constructs a DBKey that refers to the account settings for the given user: + (us:, 'account'). + """ + return DBKey(AccountSettings.ConstructSettingsId(user_id), AccountSettings.GROUP_NAME) + + @classmethod + def CreateForUser(cls, user_id, **obj_dict): + """Creates a new AccountSettings object for the given user and populates it with the given + attributes. + """ + return AccountSettings.CreateFromKeywords(settings_id=AccountSettings.ConstructSettingsId(user_id), + group_name=AccountSettings.GROUP_NAME, + user_id=user_id, + **obj_dict) + + @classmethod + def QueryByUser(cls, client, user_id, col_names, callback, must_exist=True, consistent_read=False): + """For the convenience of the caller, automatically creates the settings_id and group_name + parameters for the call to the base Query method. + """ + super(AccountSettings, cls).KeyQuery(client, + AccountSettings.ConstructKey(user_id), + col_names, + callback, + must_exist=must_exist, + consistent_read=consistent_read) diff --git a/backend/db/short_url.py b/backend/db/short_url.py new file mode 100644 index 0000000..83c20a2 --- /dev/null +++ b/backend/db/short_url.py @@ -0,0 +1,81 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Short URL support. + +This DB class stores state associated with a particular ShortURL. The state is recovered +from the database when a ShortURL link is followed. See the doc header for ShortURLBaseHandler +for more details. + +See the header for the SHORT_URL table in vf_schema.py for additional details about the table. +""" + +__authors__ = ['andy@emailscrubbed.com (Andy Kimball)'] + +import logging +import os +import time + +from tornado import gen +from viewfinder.backend.base import base64hex, constants, util +from viewfinder.backend.base.exceptions import DBConditionalCheckFailedError, TooManyRetriesError +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.guess import Guess +from viewfinder.backend.db.range_base import DBRangeObject + + +@DBObject.map_table_attributes +class ShortURL(DBRangeObject): + """Viewfinder short url data object.""" + __slots__ = [] + + KEY_LEN_IN_BYTES = 6 + assert KEY_LEN_IN_BYTES % 3 == 0 + """Size of random key, in bytes. This should always be a multiple of 3.""" + + KEY_LEN_IN_BASE64 = KEY_LEN_IN_BYTES * 4 / 3 + """Size of random key, in base-64 chars.""" + + _KEY_GEN_TRIES = 5 + """Number of tries to generate a unique key.""" + + _table = DBObject._schema.GetTable(vf_schema.SHORT_URL) + + def __init__(self, group_id=None, random_key=None): + super(ShortURL, self).__init__() + self.group_id = group_id + self.random_key = random_key + + def IsExpired(self): + """Returns true if this ShortURL has expired.""" + return util.GetCurrentTimestamp() >= self.expires + + @gen.coroutine + def Expire(self, client): + """Expires the ShortURL by setting the expires field to 0 and calling Update.""" + self.expires = 0 + yield gen.Task(self.Update, client) + + @classmethod + @gen.coroutine + def Create(cls, client, group_id, timestamp, expires, **kwargs): + """Allocate a new ShortURL DB object by finding an unused random key within the group.""" + # Try several times to generate a unique key. + for i in xrange(ShortURL._KEY_GEN_TRIES): + # Generate a random 6-byte key, using URL-safe base64 encoding. + random_key = base64hex.B64HexEncode(os.urandom(ShortURL.KEY_LEN_IN_BYTES)) + short_url = ShortURL(group_id, random_key) + short_url.timestamp = timestamp + short_url.expires = expires + short_url.json = kwargs + + try: + yield short_url.Update(client, expected={'random_key': False}) + except DBConditionalCheckFailedError as ex: + # Key is already in use, generate another. + continue + + raise gen.Return(short_url) + + logging.warning('cannot allocate a unique random key for group id "%s"', group_id) + raise TooManyRetriesError('Failed to allocate unique URL key.') diff --git a/backend/db/stopwords.py b/backend/db/stopwords.py new file mode 100644 index 0000000..2730590 --- /dev/null +++ b/backend/db/stopwords.py @@ -0,0 +1,26 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Defines full text search stop words. + +List from http://www.ranks.nl/resources/stopwords.html, which also +includes non-English stop words. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + + +STOP_WORDS = set( +"""a about above after again against all am an and any are aren't as at +be because been before being below between both but by can't cannot +could couldn't did didn't do does doesn't doing dont down during each +few for from further had hadn't has hasn't have haven't having he he'd +he'll he's her here here's hers herself him himself his how how's i id +i'll i'm i've if in into is isn't it it's its itself let's me more most +mustn't my myself no nor not of off on once only or other ought our ours +ourselves out over own same shant she she'd she'll she's should shouldn't +so some such than that that's the their theirs them themselves then there +there's these they they'd they'll they're they've this those through to +too under until up very was wasn't we we'd we'll we're we've were weren't +what what's when when's where where's which while who who's whom why why's +with won't would wouldn't you you'd you'll you're you've your yours +yourself yourselves""".split()) diff --git a/backend/db/subscription.py b/backend/db/subscription.py new file mode 100644 index 0000000..34021a5 --- /dev/null +++ b/backend/db/subscription.py @@ -0,0 +1,162 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved +"""User subscription table. + +A subscription is any time-limited modification to a user's privileges, +such as increased storage quota. Subscriptions may be paid (initially +supporting iOS in-app purchases) or granted for other reasons such as +referring new users. +""" + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +from copy import deepcopy +import time + +from viewfinder.backend.base import util +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.op.notification_manager import NotificationManager +from viewfinder.backend.services import itunes_store + +kITunesPrefix = 'itunes:' + +@DBObject.map_table_attributes +class Subscription(DBRangeObject): + """User subscription data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.SUBSCRIPTION) + + # Since our subscriptions are a combination of storage quotas and + # feature access, give each one its own product type for now. + _ITUNES_PRODUCTS = { + # vf_sub1 = "Viewfinder Plus" - cloud storage option and 5GB + 'vf_sub1': dict(product_type='vf_sub1', quantity=5), + # vf_sub2 = "Viewfinder Pro" - cloud storage, store originals, and 50GB + 'vf_sub2': dict(product_type='vf_sub2', quantity=50), + } + + _JSON_ATTRIBUTES = set(['transaction_id', 'subscription_id', 'timestamp', 'expiration_ts', 'product_type', + 'quantity', 'payment_type']) + """Subset of subscription attributes that are returned to the owning user in query_users.""" + + + @classmethod + def _GetITunesProductInfo(cls, verify_response): + """Maps iTunes product names to Subscription attributes. + + An iTunes "product" also includes information about the billing + cycle; by convention we name our products with a suffix of "_month" + or "_year" (etc). + """ + product_id = verify_response.GetProductId() + base_product, billing_cycle = product_id.rsplit('_', 1) + assert billing_cycle in ('month', 'year'), billing_cycle + return Subscription._ITUNES_PRODUCTS[base_product] + + @classmethod + def GetITunesTransactionId(cls, verify_response): + """Returns the transaction id for an iTunes transaction. + + The returned id is usable as a range key for Subscription.Query. + """ + return kITunesPrefix + verify_response.GetRenewalTransactionId() + + @classmethod + def GetITunesSubscriptionId(cls, verify_response): + """Returns the subscription id for an iTunes transaction. + + THe returned id will be the same for all transactions in a series of renewals. + """ + return kITunesPrefix + verify_response.GetOriginalTransactionId() + + @classmethod + def CreateFromITunes(cls, user_id, verify_response): + """Creates a subscription object for an iTunes transaction. + + The verify_response argument is a response from + viewfinder.backend.services.itunes_store.ITunesStoreClient.VerifyReceipt. + + The new object is returned but not saved to the database. + """ + assert verify_response.IsValid() + sub_dict = dict( + user_id=user_id, + transaction_id=Subscription.GetITunesTransactionId(verify_response), + subscription_id=Subscription.GetITunesSubscriptionId(verify_response), + timestamp=verify_response.GetTransactionTime(), + expiration_ts=verify_response.GetExpirationTime(), + payment_type='itunes', + extra_info=verify_response.GetLatestReceiptInfo(), + renewal_data=verify_response.GetRenewalData(), + ) + sub_dict.update(**Subscription._GetITunesProductInfo(verify_response)) + sub = Subscription.CreateFromKeywords(**sub_dict) + return sub + + + @classmethod + def RecordITunesTransaction(cls, client, callback, user_id, verify_response): + """Creates a subscription record for an iTunes transaction and saves it to the database. + + The verify_response argument is a response from + viewfinder.backend.services.itunes_store.ITunesStoreClient.VerifyReceipt. + """ + sub = Subscription.CreateFromITunes(user_id, verify_response) + sub.Update(client, callback) + + @classmethod + def RecordITunesTransactionOperation(cls, client, callback, user_id, verify_response_str): + def _OnRecord(): + NotificationManager.NotifyRecordSubscription(client, user_id, callback=callback) + verify_response = itunes_store.VerifyResponse.FromString(verify_response_str) + assert verify_response.IsValid() + Subscription.RecordITunesTransaction(client, _OnRecord, user_id, verify_response) + + @classmethod + def QueryByUser(cls, client, callback, user_id, include_expired=False, + include_history=False): + """Returns a list of Subscription objects for the given user. + + By default only includes currently-active subscriptions, and only + one transaction per subscription. To return expired subscriptions, + pass include_expired=True. To return all transactions (even those + superceded by a renewal transaction for the same subscription), + pass include_history=True (which implies include_expired=True). + """ + history_results = [] + latest = {} + def _VisitSub(sub, callback): + if include_history: + history_results.append(sub) + else: + if sub.expiration_ts < time.time() and not include_expired: + callback() + return + # Only one transaction per subscription. + if (sub.subscription_id in latest and + latest[sub.subscription_id].timestamp > sub.timestamp): + callback() + return + latest[sub.subscription_id] = sub + callback() + + def _OnVisitDone(): + if include_history: + assert not latest + callback(history_results) + else: + assert not history_results + callback(latest.values()) + + Subscription.VisitRange(client, user_id, None, None, _VisitSub, _OnVisitDone) + + def MakeMetadataDict(self): + """Project a subset of subscription attributes that can be provided to the user.""" + sub_dict = {} + for attr_name in Subscription._JSON_ATTRIBUTES: + util.SetIfNotNone(sub_dict, attr_name, getattr(self, attr_name, None)) + if self.extra_info: + sub_dict['extra_info'] = deepcopy(self.extra_info) + return sub_dict diff --git a/backend/db/test/__init__.py b/backend/db/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/db/test/accounting_test.py b/backend/db/test/accounting_test.py new file mode 100755 index 0000000..ebdb821 --- /dev/null +++ b/backend/db/test/accounting_test.py @@ -0,0 +1,73 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Accounting data object. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import logging +import time +import unittest + +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.op.op_context import EnterOpContext + +from base_test import DBBaseTestCase + + +class AccountingTestCase(DBBaseTestCase): + def testOperationReplay(self): + """Verify that multiple applies for the same operation ID only increment the stats once.""" + act = Accounting.CreateViewpointOwnedBy('vp1', 1) + act.num_photos = 1 + + # Manually set the current operation. + op = Operation(1, 'o1') + with EnterOpContext(op): + # First write for this entry. + self._RunAsync(Accounting.ApplyAccounting, self._client, act) + accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None) + assert accounting.StatsEqual(act) + ids = accounting.op_ids.split(',') + assert len(ids) == 1, 'len(op_ids) == %d' % len(ids) + assert op.operation_id in ids + assert accounting.num_photos == 1, 'num_photos: %d' % accounting.num_photos + + # Apply the same operation. + self._RunAsync(Accounting.ApplyAccounting, self._client, act) + accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None) + assert accounting.StatsEqual(act) + ids = accounting.op_ids.split(',') + assert len(ids) == 1, 'len(op_ids) == %d' % len(ids) + assert op.operation_id in ids + assert accounting.num_photos == 1, 'num_photos: %d' % accounting.num_photos + + # New operation. + op = Operation(1, 'o2') + with EnterOpContext(op): + self._RunAsync(Accounting.ApplyAccounting, self._client, act) + accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None) + ids = accounting.op_ids.split(',') + assert len(ids) == 2, 'len(op_ids) == %d' % len(ids) + assert op.operation_id in ids + assert accounting.num_photos == 2, 'num_photos: %d' % accounting.num_photos + + # Simulate a "repair missing accounting entry" by dbchk. This means that the stats will be there, + # but the op_ids field will be None. + accounting.op_ids = None + self._RunAsync(accounting.Update, self._client) + + # New operation. + op = Operation(1, 'o3') + with EnterOpContext(op): + self._RunAsync(Accounting.ApplyAccounting, self._client, act) + accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None) + ids = accounting.op_ids.split(',') + # op_ids is back to a size of 1. + assert len(ids) == 1, 'len(op_ids) == %d' % len(ids) + assert op.operation_id in ids + assert accounting.num_photos == 3, 'num_photos: %d' % accounting.num_photos + + self.stop() diff --git a/backend/db/test/base_test.py b/backend/db/test/base_test.py new file mode 100644 index 0000000..324fd76 --- /dev/null +++ b/backend/db/test/base_test.py @@ -0,0 +1,120 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Base TestCase class for simple, single-shard db unittests. + +Provides some pre-created objects for sub-classes. + + BaseTestCase: subclass this for all datastore object test cases +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import shutil +import tempfile +import time + +from tornado import options + +from viewfinder.backend.base import base_options +from viewfinder.backend.base import secrets, testing, util +from viewfinder.backend.db import local_client, vf_schema +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.id_allocator import IdAllocator +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.user import User +from viewfinder.backend.op.op_manager import OpManager +from viewfinder.backend.op.operation_map import DB_OPERATION_MAP +from viewfinder.backend.storage import object_store, server_log + +class DBBaseTestCase(testing.BaseTestCase): + """Base testing class for datastore objects. This test case + subclasses tornado's AsyncTestCase, which provides glue for + asynchronous tests--an IOLoop object and a means of waiting for the + completion of asynchronous events. + """ + def setUp(self): + """Sets up _client as a test emulation of DynamoDB. Creates the full + database schema, a test user, and two devices (one for mobile, one + for web-application). + """ + options.options.localdb = True + options.options.fileobjstore = True + options.options.localdb_dir = '' + + super(DBBaseTestCase, self).setUp() + options.options.localdb_dir = '' + self._client = local_client.LocalClient(vf_schema.SCHEMA) + object_store.InitObjectStore(temporary=True) + self._temp_dir = tempfile.mkdtemp() + server_log.LogBatchPersistor.SetInstance(server_log.LogBatchPersistor(backup_dir=self._temp_dir)) + IdAllocator.ResetState() + + options.options.domain = 'goviewfinder.com' + secrets.InitSecretsForTest() + + # Set deterministic testing timestamp used in place of time.time() in server code. + util._TEST_TIME = time.time() + + self._RunAsync(vf_schema.SCHEMA.VerifyOrCreate, self._client) + OpManager.SetInstance(OpManager(op_map=DB_OPERATION_MAP, client=self._client, scan_ops=True)) + + # Create users with linked email identity and default viewpoint. + self._user, self._mobile_dev = self._CreateUserAndDevice(user_dict={'user_id': 1, 'name': 'Spencer Kimball'}, + ident_dict={'key': 'Email:spencer.kimball@emailscrubbed.com', + 'authority': 'Google'}, + device_dict={'device_id': 1, 'name': 'Spencer\'s iPhone'}) + + self._user2, _ = self._CreateUserAndDevice(user_dict={'user_id': 2, 'name': 'Peter Mattis'}, + ident_dict={'key': 'Email:peter.mattis@emailscrubbed.com', + 'authority': 'Google'}, + device_dict=None) + + def tearDown(self): + """Cleanup after test is complete.""" + self._RunAsync(server_log.LogBatchPersistor.Instance().close) + self._RunAsync(OpManager.Instance().Drain) + + shutil.rmtree(self._temp_dir) + super(DBBaseTestCase, self).tearDown() + self.assertIs(Operation.GetCurrent().operation_id, None) + + def UpdateDBObject(self, cls, **db_dict): + """Update (or create if it doesn't exist) a DB object. Returns the + object. + """ + o = cls.CreateFromKeywords(**db_dict) + self._RunAsync(o.Update, self._client) + return o + + def _CreateUserAndDevice(self, user_dict, ident_dict, device_dict=None): + """Creates a new, registered user from the fields in "user_dict". If "device_dict" is + defined, then creates a new device from those fields. + + Returns a tuple containing the new user and device (if "device_dict" was given). + """ + # Allocate web device id. + webapp_dev_id = self._RunAsync(Device._allocator.NextId, self._client) + + # Create prospective user. + user, identity = self._RunAsync(User.CreateProspective, + self._client, + user_dict['user_id'], + webapp_dev_id, + ident_dict['key'], + util._TEST_TIME) + + # Register the user. + user = self._RunAsync(User.Register, + self._client, + user_dict, + ident_dict, + util._TEST_TIME, + rewrite_contacts=True) + + # Register the device. + if device_dict is not None: + device = self._RunAsync(Device.Register, self._client, user.user_id, device_dict, is_first=True) + else: + device = None + + return (user, device) diff --git a/backend/db/test/comment_test.py b/backend/db/test/comment_test.py new file mode 100644 index 0000000..bc08e44 --- /dev/null +++ b/backend/db/test/comment_test.py @@ -0,0 +1,29 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for Comment data object. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import time +import unittest + +from functools import partial + +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.comment import Comment + +from base_test import DBBaseTestCase + +class CommentTestCase(DBBaseTestCase): + def testSortOrder(self): + """Verify that comments sort ascending by timestamp.""" + timestamp = time.time() + comment_id1 = Comment.ConstructCommentId(timestamp, 0, 0) + comment_id2 = Comment.ConstructCommentId(timestamp + 1, 0, 0) + self.assertGreater(comment_id2, comment_id1) + + def testRepr(self): + comment = Comment.CreateFromKeywords(viewpoint_id='vp1', comment_id='c1', message='hello') + self.assertIn('vp1', repr(comment)) + self.assertNotIn('hello', repr(comment)) diff --git a/backend/db/test/contact_test.py b/backend/db/test/contact_test.py new file mode 100644 index 0000000..c5de0d7 --- /dev/null +++ b/backend/db/test/contact_test.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Contact. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from viewfinder.backend.base import util +from viewfinder.backend.db import versions +from viewfinder.backend.db.contact import Contact +from viewfinder.backend.db.identity import Identity + +from base_test import DBBaseTestCase + +class ContactTestCase(DBBaseTestCase): + def testUnlinkIdentity(self): + """Verify unlinking an identity causes every referencing contact to be updated.""" + # Create a Peter contact for Spencer. + timestamp = util.GetCurrentTimestamp() + spencer = self._user + contact_identity = 'Email:peter.mattis@emailscrubbed.com' + contact_name = 'Peter Mattis' + contact_given_name = 'Peter' + contact_family_name = 'Mattis' + contact_rank = 42 + contact = Contact.CreateFromKeywords(spencer.user_id, + [(contact_identity, None)], + timestamp, + Contact.GMAIL, + name='Peter Mattis', + given_name='Peter', + family_name='Mattis', + rank=42) + self._RunAsync(contact.Update, self._client) + + peter_ident = self._RunAsync(Identity.Query, self._client, contact_identity, None) + + # Unlink peter's identity, which should cause Spencer's contact to be updated. + self._RunAsync(peter_ident.UnlinkIdentity, + self._client, + self._user2.user_id, + contact_identity, + timestamp + 1) + + contacts = self._RunAsync(Contact.RangeQuery, self._client, spencer.user_id, None, None, None) + self.assertEqual(len(contacts), 1) + self.assertEqual(contacts[0].sort_key, Contact.CreateSortKey(contact.contact_id, timestamp + 1)) + self.assertEqual(contacts[0].name, contact_name) + self.assertEqual(contacts[0].given_name, contact_given_name) + self.assertEqual(contacts[0].family_name, contact_family_name) + self.assertEqual(contacts[0].rank, contact_rank) + + def testDerivedAttributes(self): + """Test that the identity and identities attributes are being properly derived from the + identities_properties attribute. + """ + # Create a Peter contact for Spencer with multiple identical and nearly identical identities. + spencer = self._user + contact_identity_a = 'Email:peter.mattis@Gmail.com' + contact_identity_b = 'Email:peterMattis@emailscrubbed.com' + contact_identity_c = 'Email:peterMattis@emailscrubbed.com' + timestamp = util.GetCurrentTimestamp() + + contact = Contact.CreateFromKeywords(spencer.user_id, + [(contact_identity_a, None), + (contact_identity_b, 'home'), + (contact_identity_c, 'work')], + timestamp, + Contact.GMAIL, + name='Peter Mattis', + given_name='Peter', + family_name='Mattis', + rank=42) + + self.assertEqual(len(contact.identities_properties), 3) + self.assertEqual(len(contact.identities), 2) + self.assertFalse(contact_identity_a in contact.identities) + self.assertFalse(contact_identity_b in contact.identities) + self.assertFalse(contact_identity_c in contact.identities) + self.assertTrue(Identity.Canonicalize(contact_identity_a) in contact.identities) + self.assertTrue(Identity.Canonicalize(contact_identity_b) in contact.identities) + self.assertTrue(Identity.Canonicalize(contact_identity_c) in contact.identities) + self.assertTrue([contact_identity_a, None] in contact.identities_properties) + self.assertTrue([contact_identity_b, 'home'] in contact.identities_properties) + self.assertTrue([contact_identity_c, 'work'] in contact.identities_properties) + + def testUnicodeContactNames(self): + """Test that contact_id generation works correctly when names include non-ascii characters.""" + name = u'ààà朋å‹ä½ å¥½abc123\U00010000\U00010000\x00\x01\b\n\t ' + + # The following will assert if there are problems when calculating the hash for the contact_id: + contact_a = Contact.CreateFromKeywords(1, + [('Email:me@my.com', None)], + util.GetCurrentTimestamp(), + Contact.GMAIL, + name=name) + + contact_b = Contact.CreateFromKeywords(1, + [('Email:me@my.com', None)], + util.GetCurrentTimestamp(), + Contact.GMAIL, + name=u'朋' + name[1:]) + + # Check that making a slight change to a unicode + self.assertNotEqual(contact_a.contact_id, contact_b.contact_id) diff --git a/backend/db/test/db_object_test.py b/backend/db/test/db_object_test.py new file mode 100644 index 0000000..3fdfd5f --- /dev/null +++ b/backend/db/test/db_object_test.py @@ -0,0 +1,118 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for DBObject. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import json +import mock +import time +import unittest + +from cStringIO import StringIO +from functools import partial +from tornado import httpclient +from viewfinder.backend.base.testing import MockAsyncHTTPClient +from viewfinder.backend.db import dynamodb_client, vf_schema +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post +from viewfinder.backend.db.test.base_test import DBBaseTestCase + + +class DBObjectTestCase(DBBaseTestCase): + @unittest.skip("needs aws credentials") + def testRangeQuery(self): + """Test DBRangeObject.RangeQuery.""" + def _MakeResponse(max_index, request): + # Enforce maximum limit of 2. + request_dict = json.loads(request.body) + limit = min(request_dict.get('Limit', 2), 2) + is_count = request_dict.get('Count') + + if 'ExclusiveStartKey' in request_dict: + start_index = int(request_dict['ExclusiveStartKey']['RangeKeyElement']['S']['S'][-1]) + 1 + else: + start_index = 0 + + count = min(max_index - start_index, limit) + items = [] + for i in xrange(start_index, start_index + count): + items.append({'ei': {'S': 'e0'}, + 'sk': {'S': 'p%d' % i}}) + + response_dict = {'Count': count, + 'ConsumedCapacityUnits': 0.5} + if not is_count: + response_dict['Items'] = items + + if start_index + count < max_index: + response_dict['LastEvaluatedKey'] = {'HashKeyElement': {'S': items[-1]['ei']}, + 'RangeKeyElement': {'S': items[-1]['sk']}} + + return httpclient.HTTPResponse(request, 200, + headers={'Content-Type': 'application/json'}, + buffer=StringIO(json.dumps(response_dict))) + + # Get session token from Amazon (no need to mock that). + client = dynamodb_client.DynamoDBClient(schema=vf_schema.SCHEMA) + self._RunAsync(client.GetItem, vf_schema.TEST_RENAME, DBKey('1', 1), attributes=None, must_exist=False) + + with mock.patch('tornado.httpclient.AsyncHTTPClient', MockAsyncHTTPClient()) as mock_client: + mock_client.map(r'https://dynamodb.us-east-1.amazonaws.com', partial(_MakeResponse, 5)) + + # Limit = None. + posts = self._RunAsync(Post.RangeQuery, client, 'e0', None, None, None) + self.assertEqual(len(posts), 2) + + # Limit = 2. + posts = self._RunAsync(Post.RangeQuery, client, 'e0', None, 2, None) + self.assertEqual(len(posts), 2) + + # Limit = 5. + posts = self._RunAsync(Post.RangeQuery, client, 'e0', None, 5, None) + self.assertEqual(len(posts), 5) + + # Limit = 7. + posts = self._RunAsync(Post.RangeQuery, client, 'e0', None, 7, None) + self.assertEqual(len(posts), 5) + + # Limit = None, count = True. + count = self._RunAsync(Post.RangeQuery, client, 'e0', None, None, None, count=True) + self.assertEqual(count, 2) + + # Limit = 2, count = True. + count = self._RunAsync(Post.RangeQuery, client, 'e0', None, 2, None, count=True) + self.assertEqual(count, 2) + + # Limit = 5, count = True. + count = self._RunAsync(Post.RangeQuery, client, 'e0', None, 5, None, count=True) + self.assertEqual(count, 5) + + # Limit = 7, count = True. + count = self._RunAsync(Post.RangeQuery, client, 'e0', None, 7, None, count=True) + self.assertEqual(count, 5) + + def testBatchQuery(self): + """Test DBObject.BatchQuery.""" + # Create some data to query. + keys = [] + for i in xrange(3): + photo_id = Photo.ConstructPhotoId(time.time(), self._mobile_dev.device_id, 1) + episode_id = Episode.ConstructEpisodeId(time.time(), self._mobile_dev.device_id, 1) + ph_dict = {'photo_id': photo_id, + 'user_id': self._user.user_id, + 'episode_id': episode_id} + self._RunAsync(Photo.CreateNew, self._client, **ph_dict) + keys.append(DBKey(photo_id, None)) + + # Add a key that will not be found. + keys.append(DBKey('unk-photo', None)) + + photos = self._RunAsync(Photo.BatchQuery, self._client, keys, None, must_exist=False) + self.assertEqual(len(photos), 4) + for i in xrange(3): + self.assertEqual(photos[i].GetKey(), keys[i]) + self.assertIsNone(photos[3]) diff --git a/backend/db/test/db_validator.py b/backend/db/test/db_validator.py new file mode 100644 index 0000000..2ff9087 --- /dev/null +++ b/backend/db/test/db_validator.py @@ -0,0 +1,891 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Validates db objects during testing. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import json + +from collections import defaultdict, namedtuple +from copy import deepcopy +from viewfinder.backend.base import message, util +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.analytics import Analytics +from viewfinder.backend.db.contact import Contact +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.followed import Followed +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.friend import Friend +from viewfinder.backend.db.id_allocator import IdAllocator +from viewfinder.backend.db.identity import Identity +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post +from viewfinder.backend.db.settings import AccountSettings +from viewfinder.backend.db.user import User +from viewfinder.backend.db.viewpoint import Viewpoint + + +DBObjectKey = namedtuple('DBObjectKey', ['db_class', 'db_key']) +"""DB objects are indexed in the model using this key.""" + + +class DBValidator(object): + """Contains a set of methods which validate the correctness of + objects in the database. + + In order to validate the correctness of the present state of an + object, it is useful to know the previous state of that object + before it was operated on. The validator achieves this by + keeping a replica of the database, called the "model". As objects + are validated, the model is kept up-to-date. Validation methods + can use the model objects in order to determine what the present + state of the database should be. + """ + def __init__(self, client, stop, wait): + self.client = client + self._stop = stop + self._wait = wait + self._model = dict() + + def Cleanup(self, validate=False): + """Deletes all objects that were created in the context of the model. + If validate=True, then validates all objects before the cleanup by + comparing them against the database. + """ + if validate: + for dbo_key, dbo in self._model.items(): + self._ValidateDBObject(dbo_key.db_class, dbo_key.db_key) + + for dbo_key, dbo in self._model.items(): + if dbo is not None: + self._RunAsync(dbo.Delete, self.client) + + + # ================================================================= + # + # Model management methods. + # + # ================================================================= + + def GetModelObject(self, cls, key, must_exist=True): + """Gets a DBObject from the model that has the specified class and key. + If "must_exist" is true, raises an exception if the object is not + present in the model. Otherwise, returns the object, or None if it is + not present in the model. + """ + if not isinstance(key, DBKey): + assert cls._table.range_key_col is None, 'key must be of type DBKey for range tables' + key = DBKey(hash_key=key, range_key=None) + + dbo_key = DBObjectKey(cls, key) + dbo = self._model.get(dbo_key, None) + assert not must_exist or dbo is not None, '%s "%s" does not exist' % (cls.__name__, key) + + return dbo + + def QueryModelObjects(self, cls, hash_key=None, predicate=None, limit=None, start_key=None, + query_forward=True): + """Returns all DBObjects from the model that: + + 1. Are of the specified class. + 2. Have the specified hash key. + 3. Match the predicate (predicate takes DBObject as argument). + 4. Have a key greater than start_key, if not None. + + Returns only up to "limit" DBObjects, and return in sorted order if + query_forward is True, or reverse sorted order otherwise. + """ + matches = [dbo + for key, dbo in self._model.items() + if dbo and key.db_class == cls + if hash_key is None or key.db_key.hash_key == hash_key + if start_key is None or (key.db_key.range_key > start_key + if query_forward else + start_key > key.db_key.range_key) + if predicate is None or predicate(dbo)] + + matches.sort(key=lambda x: x.GetKey(), reverse=not query_forward) + return matches[:limit] + + def AddModelObject(self, dbo): + """Add the specified DBObject to the list of objects tracked by the model.""" + dbo_key = DBObjectKey(dbo.__class__, dbo.GetKey()) + self._model[dbo_key] = dbo + + + # ================================================================= + # + # Simple single object validation methods. + # + # ================================================================= + + def ValidateCreateDBObject(self, cls, **db_dict): + """Validates that an object of type "cls" was created in the database, + assuming it did not already exist. If it was created, validates that + its attributes match those in "db_dict", else validates that its + attributes match those of the existing object in the model. All + attributes in "db_dict" are explicitly checked, even if they'd + normally be ignored by ValidateDBObject. Returns the object. + """ + dbo = cls.CreateFromKeywords(**db_dict) + existing_dbo = self.GetModelObject(dbo.__class__, dbo.GetKey(), must_exist=False) + if not existing_dbo: + self.AddModelObject(dbo) + existing_dbo = dbo + self._ValidateDBObject(dbo.__class__, dbo.GetKey(), must_check_dict=db_dict) + return existing_dbo + + def ValidateUpdateDBObject(self, cls, **db_dict): + """Validates that an object of type "cls" was created in the database + if it did not already exist. Validates that the attributes of the new + or existing object were updated to match those in "db_dict". All + attributes in "db_dict" are explicitly checked, even if they'd normally + be ignored by ValidateDBObject. Invokes the callback with the object. + """ + dbo = cls.CreateFromKeywords(**db_dict) + existing_dbo = self.GetModelObject(dbo.__class__, dbo.GetKey(), must_exist=False) + if not existing_dbo: + # Add the new object to the model. + self.AddModelObject(dbo) + existing_dbo = dbo + else: + # Update the existing object in the model. + existing_dbo.UpdateFromKeywords(**db_dict) + + # Now validate against the object in the database. Never ignore any updated attributes during comparison. + self._ValidateDBObject(dbo.__class__, dbo.GetKey(), must_check_dict=db_dict) + return existing_dbo + + def ValidateDeleteDBObject(self, cls, key): + """Validates that the object with the specified key was deleted from + the database. + """ + if not isinstance(key, DBKey): + assert cls._table.range_key_col is None, 'key must be of type DBKey for range tables' + key = DBKey(hash_key=key, range_key=None) + + dbo_key = DBObjectKey(cls, key) + self._model[dbo_key] = None + + self._ValidateDBObject(cls, key) + + + # ================================================================= + # + # Multiple object validation methods. + # - Shared by multiple operations + # + # ================================================================= + + def ValidateCreateContact(self, user_id, identities_properties, timestamp, contact_source, **op_dict): + """Validates creation of contact along with derived attributes. + Returns created contact. + """ + contact_dict = op_dict + contact_dict['user_id'] = user_id + contact_dict['timestamp'] = timestamp + if identities_properties is not None: + contact_dict['identities_properties'] = identities_properties + contact_dict['identities'] = set([Identity.Canonicalize(identity_properties[0]) + for identity_properties in identities_properties]) + else: + contact_dict['identities_properties'] = None + contact_dict['contact_source'] = contact_source + if 'contact_id' not in contact_dict: + contact_dict['contact_id'] = Contact.CalculateContactId(contact_dict) + if 'sort_key' not in contact_dict: + contact_dict['sort_key'] = Contact.CreateSortKey(contact_dict['contact_id'], timestamp) + + return self.ValidateCreateDBObject(Contact, **contact_dict) + + def ValidateCreateProspectiveUsers(self, op_dict, contacts): + """Validates that a prospective user has been created for any contact which is not yet + associated with Viewfinder user. Returns all resolved users. + """ + users = [] + for contact_dict in contacts: + # Look up user in the model. + if 'user_id' in contact_dict: + users.append(self.GetModelObject(User, contact_dict['user_id'])) + else: + # Look up identity and user in db in order to get various server-generated ids. + identity_key = contact_dict['identity'] + actual_ident = self._RunAsync(Identity.Query, self.client, identity_key, None) + actual_user = self._RunAsync(User.Query, self.client, actual_ident.user_id, None) + + # Determine whether the op is causing a new prospective user to be created. + expected_user = self.GetModelObject(User, actual_user.user_id, must_exist=False) + + if expected_user is None: + user_dict = {'user_id': actual_user.user_id, + 'webapp_dev_id': actual_user.webapp_dev_id} + identity_type, value = identity_key.split(':', 1) + if identity_type == 'Email': + user_dict['email'] = value + elif identity_type == 'Phone': + user_dict['phone'] = value + ident_dict = {'key': identity_key, 'authority': 'Viewfinder'} + self.ValidateUpdateUser('create prospective user', + op_dict, + user_dict, + ident_dict, + device_dict=None, + is_prospective=True) + + analytics = Analytics.Create(entity='us:%d' % actual_user.user_id, + type=Analytics.USER_CREATE_PROSPECTIVE) + self.ValidateCreateDBObject(Analytics, **analytics._asdict()) + + users.append(actual_user) + + return users + + def ValidateFollower(self, user_id, viewpoint_id, labels, last_updated, + timestamp=None, adding_user_id=None, viewed_seq=None): + """Validates that Follower and Followed records have been created or updated in the database + for user "user_id" and viewpoint "viewpoint_id". + + Returns the follower. + """ + follower_dict = {'user_id': user_id, + 'viewpoint_id': viewpoint_id, + 'labels': labels} + util.SetIfNotNone(follower_dict, 'timestamp', timestamp) + util.SetIfNotNone(follower_dict, 'adding_user_id', adding_user_id) + util.SetIfNotNone(follower_dict, 'viewed_seq', viewed_seq) + follower = self.ValidateUpdateDBObject(Follower, **follower_dict) + + self._ValidateUpdateFollowed(user_id, viewpoint_id, None, last_updated) + + return follower + + def ValidateUpdateUser(self, name, op_dict, user_dict, ident_dict, + device_dict=None, is_prospective=False): + """Validates that a user and identity have been created in the database + if they did not already exist, or were updated if they did. If + "device_dict" is defined, validates that a device was created or updated + as well. + """ + user_id = user_dict['user_id'] + + # Validate creation of the default viewpoint, follower, and followed record. + viewpoint_id = Viewpoint.ConstructViewpointId(user_dict['webapp_dev_id'], 0) + viewpoint = self.GetModelObject(User, user_id, must_exist=False) + if viewpoint is None: + expected_viewpoint = self.ValidateCreateDBObject(Viewpoint, + viewpoint_id=viewpoint_id, + user_id=user_id, + timestamp=op_dict['op_timestamp'], + last_updated=op_dict['op_timestamp'], + type=Viewpoint.DEFAULT, + update_seq=0) + + labels = Follower.PERMISSION_LABELS + [Follower.PERSONAL] + expected_follower = self.ValidateFollower(user_id=user_id, + viewpoint_id=viewpoint_id, + timestamp=op_dict['op_timestamp'], + labels=labels, + last_updated=op_dict['op_timestamp'], + viewed_seq=0) + + # Validate User object. + scratch_user_dict = deepcopy(user_dict) + if ident_dict.get('authority', None) == 'Facebook' and user_dict.get('email', None): + scratch_user_dict['facebook_email'] = user_dict['email'] + + union_label = [] if is_prospective else [User.REGISTERED] + existing_user = self.GetModelObject(User, user_id, must_exist=False) + if existing_user is None: + is_registering = False + before_user_dict = None + scratch_user_dict['private_vp_id'] = viewpoint_id + scratch_user_dict['labels'] = union_label + else: + is_registering = not existing_user.IsRegistered() + before_user_dict = existing_user._asdict() + scratch_user_dict.update(before_user_dict) + scratch_user_dict['labels'] = list(set(scratch_user_dict['labels']).union(union_label)) + + expected_user = self.ValidateUpdateDBObject(User, **scratch_user_dict) + + # Validate AccountSettings object. + settings = AccountSettings.CreateForUser(user_id) + + if device_dict is None: + if self.GetModelObject(AccountSettings, settings.GetKey(), must_exist=False) is None: + # First web device was registered, so validate that emails or sms messages are turned on. + settings.push_alerts = AccountSettings.PUSH_NONE + settings.email_alerts = AccountSettings.EMAIL_NONE + settings.sms_alerts = AccountSettings.SMS_NONE + + identity_type, identity_value = Identity.SplitKey(ident_dict['key']) + if identity_type == 'Email': + settings.email_alerts = AccountSettings.EMAIL_ON_SHARE_NEW + elif identity_type == 'Phone': + settings.sms_alerts = AccountSettings.SMS_ON_SHARE_NEW + else: + if len(self.QueryModelObjects(Device, user_id)) == 0: + # First mobile device was registered, so validate that emails and sms messages are + # turned off and push alerts turned on. + settings.push_alerts = AccountSettings.PUSH_ALL + settings.email_alerts = AccountSettings.EMAIL_NONE + settings.sms_alerts = AccountSettings.SMS_NONE + + self.ValidateUpdateDBObject(AccountSettings, **settings._asdict()) + + # Validate Friend object. + self.ValidateUpdateDBObject(Friend, user_id=user_id, friend_id=user_id) + + # Validate Identity object. + existing_identity = self.GetModelObject(Identity, ident_dict['key'], must_exist=False) + expected_ident = self.ValidateUpdateDBObject(Identity, + user_id=user_id, + **ident_dict) + + # Validate Device object. + if device_dict is not None: + update_dict = {'user_id': user_id, + 'timestamp': util._TEST_TIME, + 'last_access': util._TEST_TIME} + if 'push_token' in device_dict: + update_dict['alert_user_id'] = user_id + update_dict.update(device_dict) + expected_device = self.ValidateUpdateDBObject(Device, **update_dict) + + # Validate that any other devices with same push token have had their tokens revoked. + if 'push_token' in device_dict: + predicate = lambda d: d.device_id != expected_device.device_id and d.push_token == expected_device.push_token + other_devices = self.QueryModelObjects(Device, predicate=predicate) + for device in other_devices: + self.ValidateUpdateDBObject(Device, + user_id=device.user_id, + device_id=device.device_id, + push_token=None, + alert_user_id=None) + + # Validate Contact objects. + if existing_identity is None or is_registering: + self.ValidateRewriteContacts(expected_ident.key, op_dict) + + # Validate contact notifications. + self.ValidateContactNotifications(name, expected_ident.key, op_dict) + + # Validate Friend notifications. + after_user_dict = self.GetModelObject(User, user_id)._asdict() + if before_user_dict != after_user_dict and not is_prospective: + invalidate = {'users': [user_id]} + self.ValidateFriendNotifications('register friend', user_id, op_dict, invalidate) + + # Validate analytics entry for Register. + if existing_user is None: + # User is being created for the first time, it must have a CREATE_PROSPECTIVE analytics entry. + analytics = Analytics.Create(entity='us:%d' % user_id, + type=Analytics.USER_CREATE_PROSPECTIVE) + self.ValidateCreateDBObject(Analytics, **analytics._asdict()) + + if (not existing_user or is_registering) and not is_prospective: + # User is being registered. + analytics = Analytics.Create(entity='us:%d' % user_id, + type=Analytics.USER_REGISTER) + self.ValidateCreateDBObject(Analytics, **analytics._asdict()) + + def ValidateRewriteContacts(self, identity_key, op_dict): + """Validates that all contacts that reference "identity_key" have been updated with the + new timestamp. + """ + # Iterate over all contacts that reference the identity. + for co in self.QueryModelObjects(Contact, predicate=lambda co: identity_key in co.identities): + # Validate that contact has been updated. + contact_dict = co._asdict() + contact_dict['timestamp'] = op_dict['op_timestamp'] + sort_key = Contact.CreateSortKey(Contact.CalculateContactId(contact_dict), contact_dict['timestamp']) + contact_dict['sort_key'] = sort_key + self.ValidateUpdateDBObject(Contact, **contact_dict) + + # Validate that any old contacts have been deleted. + if sort_key != co.sort_key: + self.ValidateDeleteDBObject(Contact, co.GetKey()) + + def ValidateFriendsInGroup(self, user_ids): + """Validates that all specified users are friends with each other.""" + for index, user_id in enumerate(user_ids): + for friend_id in user_ids[index + 1:]: + if user_id != friend_id: + user1 = self.GetModelObject(User, user_id) + user2 = self.GetModelObject(User, friend_id) + + self.ValidateUpdateDBObject(Friend, user_id=user_id, friend_id=friend_id) + self.ValidateUpdateDBObject(Friend, user_id=friend_id, friend_id=user_id) + + def ValidateCopyEpisodes(self, op_dict, viewpoint_id, ep_dicts): + """Validates that a set of episodes and posts have been created within the specified + viewpoint via a sharing or save operation. + """ + ph_act_dict = {} + for ep_dict in ep_dicts: + existing_episode = self.GetModelObject(Episode, ep_dict['existing_episode_id']) + + new_ep_dict = {'episode_id': ep_dict['new_episode_id'], + 'parent_ep_id': ep_dict['existing_episode_id'], + 'user_id': op_dict['user_id'], + 'viewpoint_id': viewpoint_id, + 'timestamp': existing_episode.timestamp, + 'publish_timestamp': op_dict['op_timestamp'], + 'location': existing_episode.location, + 'placemark': existing_episode.placemark} + self.ValidateCreateDBObject(Episode, **new_ep_dict) + + for photo_id in ep_dict['photo_ids']: + post = self.GetModelObject(Post, DBKey(ep_dict['new_episode_id'], photo_id), must_exist=False) + if post is None or post.IsRemoved(): + if post is None: + self.ValidateCreateDBObject(Post, episode_id=ep_dict['new_episode_id'], photo_id=photo_id) + else: + self.ValidateUpdateDBObject(Post, episode_id=ep_dict['new_episode_id'], photo_id=photo_id, labels=[]) + ph_act_dict.setdefault(viewpoint_id, {}).setdefault(ep_dict['new_episode_id'], []).append(photo_id) + + def ValidateCoverPhoto(self, viewpoint_id, unshare_ep_dicts=None): + """Validate that a cover_photo is set on a viewpoint. Selects a new cover_photo if + there currently isn't one or the current one is no longer shared in the viewpoint. + Returns: True if viewpoint's cover_photo value changed, otherwise False. + """ + current_model_cover_photo = self.GetModelObject(Viewpoint, viewpoint_id).cover_photo + + exclude_posts_set = set() + if unshare_ep_dicts is not None: + exclude_posts_set = set([(episode_id, photo_id) + for (episode_id, photo_ids) in unshare_ep_dicts.items() + for photo_id in photo_ids]) + elif current_model_cover_photo is not None: + # No unshares and a cover photo is already set, so we don't do anything. + return False + + # Take a shortcut here and call the implementation to select the photo. If we start + # seeing bugs in this area, we may want to do a test model implementation. + # We use the actual db because that already has all the activities written to it that + # SelectCoverPhoto() depends on. + viewpoint = self._RunAsync(Viewpoint.Query, self.client, viewpoint_id, col_names=None) + selected_cover_photo = self._RunAsync(viewpoint.SelectCoverPhoto, self.client, exclude_posts_set) + + if current_model_cover_photo != selected_cover_photo: + # Update cover_photo with whatever we've got at this point. + self.ValidateUpdateDBObject(Viewpoint, + viewpoint_id=viewpoint_id, + cover_photo=selected_cover_photo) + return current_model_cover_photo != selected_cover_photo + + def ValidateUnlinkIdentity(self, op_dict, identity_key): + """Validates that the specified identity was properly unlinked from the attached user.""" + identity = self.GetModelObject(Identity, identity_key, must_exist=False) + + # Validate that the identity has been removed. + self.ValidateDeleteDBObject(Identity, identity_key) + + # Validate that all contacts have been unlinked from the user. + self.ValidateRewriteContacts(identity_key, op_dict) + + # Validate contact notifications. + self.ValidateContactNotifications('unlink_identity', identity_key, op_dict) + + # Validate user notification for the owner of the identity (if it existed). + if identity: + self.ValidateUserNotification('unlink_self', op_dict['user_id'], op_dict) + + def ValidateReviveRemovedFollowers(self, viewpoint_id, op_dict): + """Validates that the REMOVED followers label has been removed from viewpoint followers. + Removed followers should be revived by any structural changes to their viewpoints. + """ + follower_matches = lambda f: f.viewpoint_id == viewpoint_id + for follower in self.QueryModelObjects(Follower, predicate=follower_matches): + if follower.IsRemoved() and not follower.IsUnrevivable(): + follower.labels.remove(Follower.REMOVED) + follower.labels = follower.labels.combine() + self.ValidateUpdateDBObject(Follower, + user_id=follower.user_id, + viewpoint_id=follower.viewpoint_id, + labels=follower.labels) + + self.ValidateNotification('revive followers', + follower.user_id, + op_dict, + DBValidator.CreateViewpointInvalidation(viewpoint_id), + viewpoint_id=viewpoint_id) + + def ValidateTerminateAccount(self, user_id, op_dict, merged_with=None): + """Validates that the given user's account has been terminated.""" + # Validate that all alerts have been stopped to user devices. + devices = self.QueryModelObjects(Device, predicate=lambda d: d.user_id == user_id) + for device in devices: + self.ValidateUpdateDBObject(Device, + user_id=user_id, + device_id=device.device_id, + alert_user_id=None) + + # Validate that all identities attached to the user have been unlinked. + identities = self.QueryModelObjects(Identity, predicate=lambda i: i.user_id == user_id) + for identity in identities: + self.ValidateUnlinkIdentity(op_dict, identity.key) + + # Validate that "terminated" label is added to User object. + user = self.GetModelObject(User, user_id) + labels = user.labels.union([User.TERMINATED]) + self.ValidateUpdateDBObject(User, user_id=user_id, labels=labels, merged_with=merged_with) + + # Validate notifications to friends of the user account. + invalidate = {'users': [user_id]} + self.ValidateFriendNotifications('terminate_account', user_id, op_dict, invalidate) + + # Validate analytics entry. + analytics = Analytics.Create(entity='us:%d' % user_id, + type=Analytics.USER_TERMINATE) + self.ValidateCreateDBObject(Analytics, **analytics._asdict()) + + def ValidateAccounting(self): + """Validates accounting stats for all viewpoints and users.""" + desired_act = {} + + def _SetOrIncrement(act): + key = (act.hash_key, act.sort_key) + if key not in desired_act: + desired_act[key] = act + else: + desired_act[key].IncrementStatsFrom(act) + + followers = defaultdict(list) + all_followers, _ = self._RunAsync(Follower.Scan, self.client, None) + for follower in all_followers: + followers[follower.viewpoint_id].append(follower) + + all_viewpoints, _ = self._RunAsync(Viewpoint.Scan, self.client, None) + for viewpoint in all_viewpoints: + # Validate the viewpoint accounting stats. + self.ValidateViewpointAccounting(viewpoint.viewpoint_id) + + # Get accounting for the viewpoint. + vp_vt_act = Accounting.CreateViewpointVisibleTo(viewpoint.viewpoint_id) + vp_vt_act = self.GetModelObject(Accounting, vp_vt_act.GetKey(), must_exist=False) or vp_vt_act + + # Increment user-level accounting stats. + for follower in followers[viewpoint.viewpoint_id]: + if not follower.IsRemoved(): + # Add to follower's accounting. + us_vt_act = Accounting.CreateUserVisibleTo(follower.user_id) + us_vt_act.CopyStatsFrom(vp_vt_act) + _SetOrIncrement(us_vt_act) + + vp_sb_act = Accounting.CreateViewpointSharedBy(viewpoint.viewpoint_id, follower.user_id) + vp_sb_act = self.GetModelObject(Accounting, vp_sb_act.GetKey(), must_exist=False) or vp_sb_act + + us_sb_act = Accounting.CreateUserSharedBy(follower.user_id) + us_sb_act.CopyStatsFrom(vp_sb_act) + _SetOrIncrement(us_sb_act) + + for key, value in desired_act.iteritems(): + # Absence of accounting record is equivalent to zero-value accounting record. + if value.IsZero(): + act = self._RunAsync(Accounting.KeyQuery, self.client, value.GetKey(), None, must_exist=False) + if act is None: + continue + + self.ValidateUpdateDBObject(Accounting, **value._asdict()) + + def ValidateViewpointAccounting(self, viewpoint_id): + """Validates the given viewpoint's accounting stats by iterating over all viewable photos + and adding up the expected stats for each. + """ + desired_act = {} + + def _SetOrIncrement(act): + key = (act.hash_key, act.sort_key) + if key not in desired_act: + desired_act[key] = act + else: + desired_act[key].IncrementStatsFrom(act) + + viewpoint = self._RunAsync(Viewpoint.Query, self.client, viewpoint_id, None) + episodes, _ = self._RunAsync(Viewpoint.QueryEpisodes, self.client, viewpoint_id) + for episode in episodes: + act = Accounting() + posts = self._RunAsync(Post.RangeQuery, self.client, episode.episode_id, None, None, None) + for post in posts: + if post.IsRemoved(): + continue + + photo = self._RunAsync(Photo.Query, self.client, post.photo_id, None) + act.IncrementFromPhotos([photo]) + + # Get follower record of owner of the episode. + follower = self._RunAsync(Follower.Query, self.client, episode.user_id, viewpoint_id, None) + + if viewpoint.IsDefault(): + # Default viewpoint. only "owned by" types are filled in. + if not follower.IsRemoved(): + vp_ob_act = Accounting.CreateViewpointOwnedBy(viewpoint_id, episode.user_id) + vp_ob_act.CopyStatsFrom(act) + _SetOrIncrement(vp_ob_act) + + us_ob_act = Accounting.CreateUserOwnedBy(episode.user_id) + us_ob_act.CopyStatsFrom(act) + _SetOrIncrement(us_ob_act) + else: + # Shared viewpoint - fill in "shared by" and "visible to". + vp_sb_act = Accounting.CreateViewpointSharedBy(viewpoint_id, episode.user_id) + vp_sb_act.CopyStatsFrom(act) + _SetOrIncrement(vp_sb_act) + + vp_vt_act = Accounting.CreateViewpointVisibleTo(viewpoint_id) + vp_vt_act.CopyStatsFrom(act) + _SetOrIncrement(vp_vt_act) + + for key, value in desired_act.iteritems(): + # Absence of accounting record is equivalent to zero-value accounting record. + if value.IsZero() and self.GetModelObject(Accounting, value.GetKey(), must_exist=False) == None: + continue + self.ValidateUpdateDBObject(Accounting, **value._asdict()) + + def ValidateFollowerNotifications(self, viewpoint_id, activity_dict, op_dict, invalidate, sends_alert=False): + """Validates that a notification has been created for each follower of the specified + viewpoint. If "invalidate" is a dict, then each follower uses that as its invalidation. + Otherwise, it is assumed to be a func that returns the invalidation, given the id of the + follower. Validates that an activity was created in the viewpoint. + + The "activity_dict" must contain expected "activity_id", "timestamp", and "name" fields. + The "op_dict" must contain expected "user_id", "device_id", "op_id", and "op_timestamp" + fields. + """ + # Validate that last_updated and update_seq have been properly updated. + viewpoint = self.GetModelObject(Viewpoint, viewpoint_id) + old_timestamp = viewpoint.last_updated + new_timestamp = max(old_timestamp, op_dict['op_timestamp']) + update_seq = 1 if viewpoint.update_seq is None else viewpoint.update_seq + 1 + self.ValidateUpdateDBObject(Viewpoint, + viewpoint_id=viewpoint_id, + last_updated=new_timestamp, + update_seq=update_seq) + + # Validate new Activity object. + activity_dict = deepcopy(activity_dict) + activity = self.ValidateCreateDBObject(Activity, + viewpoint_id=viewpoint_id, + activity_id=activity_dict.pop('activity_id'), + timestamp=activity_dict.pop('timestamp'), + name=activity_dict.pop('name'), + user_id=op_dict['user_id'], + update_seq=update_seq, + json=util.ToCanonicalJSON(activity_dict)) + + # Validate revival of removed followers in most cases. + if activity.name not in ['unshare', 'remove_followers']: + self.ValidateReviveRemovedFollowers(viewpoint_id, op_dict) + + # Validate that a notification was created for each follower. + follower_matches = lambda f: f.viewpoint_id == viewpoint_id + for follower in self.QueryModelObjects(Follower, predicate=follower_matches): + if not follower.IsRemoved() or activity.name in ['remove_followers']: + # Validate that sending follower had its viewed_seq field incremented. + if follower.user_id == op_dict['user_id']: + viewed_seq = 1 if follower.viewed_seq is None else follower.viewed_seq + 1 + self.ValidateUpdateDBObject(Follower, + user_id=follower.user_id, + viewpoint_id=viewpoint_id, + viewed_seq=viewed_seq) + else: + viewed_seq = None + + # Validate that the Followed row for this follower had its timestamp updated. + self._ValidateUpdateFollowed(follower.user_id, viewpoint_id, old_timestamp, new_timestamp) + + if invalidate is None or isinstance(invalidate, dict): + foll_invalidate = invalidate + else: + foll_invalidate = invalidate(follower.user_id) + + # Validate that a new notification was created for this follower. + self.ValidateNotification(activity.name, + follower.user_id, + op_dict, + foll_invalidate, + activity_id=activity.activity_id, + viewpoint_id=viewpoint_id, + sends_alert=sends_alert, + seq_num_pair=(update_seq, viewed_seq)) + + def ValidateFriendNotifications(self, name, user_id, op_dict, invalidate): + """Validates that a notification was created for each friend of the given user, as well as + the user himself. + """ + for friend in self.QueryModelObjects(Friend, predicate=lambda fr: fr.user_id == user_id): + self.ValidateNotification(name, friend.friend_id, op_dict, invalidate) + + def ValidateContactNotifications(self, name, identity_key, op_dict): + """Validates that a notification was created for each user who + references a contact of the specified identity. + """ + invalidate = {'contacts': {'start_key': Contact.CreateSortKey(None, op_dict['op_timestamp'])}} + for co in self.QueryModelObjects(Contact, predicate=lambda co: identity_key in co.identities): + self.ValidateNotification(name, co.user_id, op_dict, invalidate) + + def ValidateUserNotification(self, name, user_id, op_dict): + """Validates that a notification was created for the specified user.""" + self.ValidateNotification(name, user_id, op_dict, {'users': [user_id]}) + + def ValidateNotification(self, name, target_user_id, op_dict, invalidate, + activity_id=None, viewpoint_id=None, seq_num_pair=None, sends_alert=False): + """Validates that a notification with the specified name and invalidation + has been created for "target_user_id". + + The "op_dict" must contain expected "user_id", "device_id", and + "op_timestamp" fields. It may contain "op_id", if its expected value + is known to the caller. + """ + # Validate that new notification is based on previous notification. + notifications = self.QueryModelObjects(Notification, target_user_id) + last_notification = notifications[-1] if notifications else None + + notification_id = last_notification.notification_id + 1 if last_notification is not None else 1 + if invalidate is not None: + invalidate = deepcopy(invalidate) + invalidate['headers'] = dict(version=message.MAX_MESSAGE_VERSION) + invalidate = json.dumps(invalidate) + + badge = last_notification.badge if last_notification is not None else 0 + if sends_alert and target_user_id != op_dict['user_id']: + badge += 1 + + self.ValidateCreateDBObject(Notification, + notification_id=notification_id, + user_id=target_user_id, + name=name, + timestamp=op_dict['op_timestamp'], + sender_id=op_dict['user_id'], + sender_device_id=op_dict['device_id'], + invalidate=invalidate, + badge=badge, + activity_id=activity_id, + viewpoint_id=viewpoint_id, + update_seq=None if seq_num_pair is None else seq_num_pair[0], + viewed_seq=None if seq_num_pair is None else seq_num_pair[1]) + + # Optionally validate the op_id field, if it was passed in op_dict. + if 'op_id' in op_dict: + self.ValidateUpdateDBObject(Notification, + notification_id=notification_id, + user_id=target_user_id, + op_id=op_dict['op_id']) + + @classmethod + def CreateViewpointInvalidation(cls, viewpoint_id): + """Create invalidation for entire viewpoint, including all metadata and all collections. + + NOTE: Make sure to update this when new viewpoint collections are added. + """ + return {'viewpoints': [{'viewpoint_id': viewpoint_id, + 'get_attributes': True, + 'get_followers': True, + 'get_activities': True, + 'get_episodes': True, + 'get_comments': True}]} + + + # ================================================================= + # + # Helper methods. + # + # ================================================================= + + def _ValidateUpdateFollowed(self, user_id, viewpoint_id, old_timestamp, new_timestamp): + """Validate that an older Followed record was deleted and a newer created.""" + if old_timestamp is not None and \ + Followed._TruncateToDay(new_timestamp) > Followed._TruncateToDay(old_timestamp): + db_key = DBKey(user_id, Followed.CreateSortKey(viewpoint_id, old_timestamp)) + self.ValidateDeleteDBObject(Followed, db_key) + + self.ValidateCreateDBObject(Followed, + user_id=user_id, + viewpoint_id=viewpoint_id, + sort_key=Followed.CreateSortKey(viewpoint_id, new_timestamp), + date_updated=Followed._TruncateToDay(new_timestamp)) + + def _RunAsync(self, func, *args, **kwargs): + """Runs an async function which takes a callback argument. Waits for + the function to complete and returns any result. + """ + func(callback=self._stop, *args, **kwargs) + return self._wait() + + def _ValidateDBObject(self, cls, key, must_check_dict=None): + """Validates that a model object of type "cls", and with the specified + key is equivalent to the actual DBObject that exists (or not) in the + database. Always checks attributes in "must_check_dict", even if + normally the attribute would be ignored. + """ + # Get the expected object from the model. + expected_dbo = self.GetModelObject(cls, key, must_exist=False) + if expected_dbo is not None: + expected_dict = self._SanitizeDBObject(expected_dbo, must_check_dict) + expected_json = util.ToCanonicalJSON(expected_dict, indent=True) + else: + expected_json = None + + # Get the actual object to validate from the database. + actual_dbo = self._RunAsync(cls.KeyQuery, self.client, key, None, must_exist=False) + if actual_dbo is not None: + actual_dict = self._SanitizeDBObject(actual_dbo, must_check_dict) + actual_json = util.ToCanonicalJSON(actual_dict, indent=True) + else: + actual_json = None + + if expected_json != actual_json: + # Special-case notifications in order to show all notifications for the user to aid in debugging. + if cls == Notification and expected_dbo is not None: + expected_seq = self.QueryModelObjects(Notification, expected_dbo.user_id) + actual_seq = self._RunAsync(Notification.RangeQuery, self.client, expected_dbo.user_id, None, None, None) + + raise AssertionError("DBObject difference detected.\n\nEXPECTED (%s): %s\n%s\n\nACTUAL (%s): %s\n%s\n" % \ + (type(expected_dbo).__name__, expected_json, [n.name for n in expected_seq], + type(actual_dbo).__name__, actual_json, [n.name for n in actual_seq])) + else: + raise AssertionError("DBObject difference detected.\n\nEXPECTED (%s): %s\n\nACTUAL (%s): %s\n" % \ + (type(expected_dbo).__name__, expected_json, type(actual_dbo).__name__, actual_json)) + + def _SanitizeDBObject(self, dbo, must_check_dict): + """Converts dbo to a dict, and then removes attributes from it that + should be ignored when comparing DBObjects with each other. + """ + dbo_dict = dbo._asdict() + + remove_dict = {Accounting: set(['op_ids']), + AccountSettings: set(['sms_count']), + Analytics: set(['payload']), + Follower: set(['viewed_seq']), + IdAllocator: set(['next_id']), + Identity: set(['last_fetch', 'token_guesses', 'token_guesses_time', 'json_attrs', 'auth_throttle']), + Notification: set(['op_id']), + User: set(['asset_id_seq', 'signing_key', 'pwd_hash', 'salt']), + Viewpoint: set(['last_updated', 'update_seq'])} + + for key, value in dbo_dict.items(): + # Always remove _version attribute. + dbo_dict.pop('_version', None) + + # Normalize any lists. + if isinstance(value, list): + dbo_dict[key] = sorted(value) + + # Normalize the order of attributes embedded in json_attr. + if key == 'json' or key == 'invalidate': + json_dict = json.loads(value) + json_dict.pop('headers', None) + dbo_dict[key] = util.ToCanonicalJSON(json_dict) + + # Remove certain attributes from certain types. + remove_set = remove_dict.get(type(dbo), []) + if key in remove_set and (must_check_dict is None or key not in must_check_dict): + del dbo_dict[key] + + return dbo_dict diff --git a/backend/db/test/db_versions_test.py b/backend/db/test/db_versions_test.py new file mode 100644 index 0000000..f39b494 --- /dev/null +++ b/backend/db/test/db_versions_test.py @@ -0,0 +1,109 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Versions class. +""" + +__author__ = 'andy@emailscrubbed.comm (Andy Kimball)' + +from tornado import options +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.test.base_test import DBBaseTestCase +from viewfinder.backend.db.test.test_rename import TestRename +from viewfinder.backend.db.versions import Version, TEST_VERSION, TEST_VERSION2 +from viewfinder.backend.db.tools import upgrade + + +class DbVersionsTestCase(DBBaseTestCase): + def setUp(self): + super(DbVersionsTestCase, self).setUp() + + def _CreateTestObject(**test_dict): + o = TestRename() + o.UpdateFromKeywords(**test_dict) + o.Update(self._client, self.stop) + self.wait() + + # Create some rows for the tests. + test_dict = { + 'test_hk': 't1', + 'test_rk': 1, + 'attr1': 1000, + 'attr2': 'hello world', + '_version': 0 + } + _CreateTestObject(**test_dict) + + test_dict['test_rk'] = 2 + del test_dict['attr1'] + _CreateTestObject(**test_dict) + + # Should only run one migrator. + test_dict['test_rk'] = 3 + test_dict['_version'] = TEST_VERSION.rank + _CreateTestObject(**test_dict) + + test_dict['test_rk'] = 4 + test_dict['attr1'] = 2000 + test_dict['_version'] = Version.GetCurrentVersion() + _CreateTestObject(**test_dict) + + def testMaybeMigrate(self): + """Explicit migration.""" + obj_list, _ = self._RunAsync(TestRename.Scan, self._client, col_names=None) + for obj in obj_list: + # Migrate object. + self._RunAsync(Version.MaybeMigrate, self._client, obj, [TEST_VERSION, TEST_VERSION2]) + + # Now, re-query and validate. + obj_list, _ = self._RunAsync(TestRename.Scan, self._client, col_names=None) + self._Validate(obj_list[0],obj_list[1],obj_list[2],obj_list[3]) + + def testUpgradeTool(self): + """Test the upgrade.py tool against the TestRename table.""" + options.options.migrator = 'TEST_VERSION' + self._RunAsync(upgrade.UpgradeTable, self._client, TestRename._table) + options.options.migrator = 'TEST_VERSION2' + self._RunAsync(upgrade.UpgradeTable, self._client, TestRename._table) + + # Verify that the upgrades happened. + list = self._RunAsync(TestRename.RangeQuery, self._client, 't1', range_desc=None, limit=None, col_names=None) + self._Validate(list[0], list[1], list[2], list[3]) + + @async_test + def testNoMutation(self): + """Test migration with mutations turned off.""" + def _OnQuery(o): + Version.SetMutateItems(True) + assert o.attr0 is None, o.attr0 + assert o.attr1 == 1000, o.attr1 + assert o._version == 0, o._version + self.stop() + + def _OnUpgrade(): + TestRename.KeyQuery(self._client, DBKey(hash_key='t1', range_key=1), + col_names=['attr0', 'attr1'], callback=_OnQuery) + + Version.SetMutateItems(False) + options.options.migrator = 'TEST_VERSION' + upgrade.UpgradeTable(self._client, TestRename._table, _OnUpgrade) + + def _Validate(self, test_obj, test_obj2=None, test_obj3=None, test_obj4=None): + assert test_obj.attr0 == 100, test_obj.attr0 + assert test_obj.attr1 is None, test_obj.attr1 + assert test_obj._version >= TEST_VERSION2.rank, test_obj._version + + if test_obj2: + assert test_obj2.attr0 == 100, test_obj2.attr0 + assert test_obj2.attr1 is None, test_obj2.attr1 + assert test_obj2._version >= TEST_VERSION2.rank, test_obj2._version + + if test_obj3: + assert test_obj3.attr0 is None, test_obj3.attr0 + assert test_obj3.attr1 is None, test_obj3.attr1 + assert test_obj3._version >= TEST_VERSION2.rank, test_obj3._version + + if test_obj4: + assert test_obj4.attr0 is None, test_obj4.attr0 + assert test_obj4.attr1 == 2000, test_obj4.attr1 + assert test_obj4._version >= TEST_VERSION2.rank, test_obj4._version diff --git a/backend/db/test/dbchk_test.py b/backend/db/test/dbchk_test.py new file mode 100644 index 0000000..fa86897 --- /dev/null +++ b/backend/db/test/dbchk_test.py @@ -0,0 +1,905 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for dbchk tool. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import time + +from tornado import options +from viewfinder.backend.base import constants, util +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.comment import Comment +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.followed import Followed +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.job import Job +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post +from viewfinder.backend.db.test.db_validator import DBValidator +from viewfinder.backend.db.tools import dbchk +from viewfinder.backend.db.viewpoint import Viewpoint +from viewfinder.backend.op.notification_manager import NotificationManager + +from base_test import DBBaseTestCase + +class DbChkTestCase(DBBaseTestCase): + def setUp(self): + super(DbChkTestCase, self).setUp() + + self._start_test_time = util._TEST_TIME + self._validator = DBValidator(self._client, self.stop, self.wait) + + dbchk._TEST_MODE = True + self._checker = dbchk.DatabaseChecker(self._client) + self.maxDiff = 2048 + + # Create op_dict, using same dummy operation values as used in dbchk.py. + self._op_dict = {'op_id': Operation.ConstructOperationId(Operation.ANONYMOUS_DEVICE_ID, 0), + 'op_timestamp': util._TEST_TIME, + 'user_id': Operation.ANONYMOUS_USER_ID, + 'device_id': Operation.ANONYMOUS_DEVICE_ID} + + def tearDown(self): + # Restore default dbchk options. + options.options.viewpoints = [] + options.options.repair = False + options.options.email = 'dbchk@emailscrubbed.com' + + super(DbChkTestCase, self).tearDown() + + def testMultipleCorruptions(self): + """Verifies detection of multiple corruption issues.""" + # Create 2 empty viewpoints without Followed records. + self._CreateTestViewpoint('vp1', self._user.user_id, [], delete_followed=True) + self._CreateTestViewpoint('vp2', self._user.user_id, [], delete_followed=True) + + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' missing followed (1 instance)\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' missing followed (1 instance)\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1,vp2' + + self.assertEqual(self._checker._email_args, + {'fromname': 'DB Checker', + 'text': 'Found corruption(s) in database:\n\n%s' % corruption_text, + 'subject': 'Database corruption', + 'from': 'dbchk@emailscrubbed.com', + 'to': 'dbchk@emailscrubbed.com'}) + + # Check again, but with last_scan value that should find nothing new. + self._RunAsync(self._checker.CheckAllViewpoints, last_scan=time.time() + 1) + self.assertIsNone(self._checker._email_args) + + def testEmail(self): + """Verifies the --email command line option.""" + self._CreateTestViewpoint('vp1', self._user.user_id, [], delete_followed=True) + + options.options.email = 'kimball.andy@emailscrubbed.com' + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' missing followed (1 instance)\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args, + {'fromname': 'DB Checker', + 'text': 'Found corruption(s) in database:\n\n%s' % corruption_text, + 'subject': 'Database corruption', + 'from': 'dbchk@emailscrubbed.com', + 'to': 'kimball.andy@emailscrubbed.com'}) + + def testExclusiveLock(self): + """Test running dbchk with locking.""" + # Create empty non-default viewpoints (already have empty default viewpoints). + self._CreateTestViewpoint('vp1', self._user.user_id, []) + + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + # Grab lock and run in repair mode. There should still be errors. + lock = self._RunAsync(Lock.Acquire, self._client, LockResourceType.Job, 'dbchk', 'dbck') + assert lock is not None + + self._RunDbChk({'viewpoints': ['vp1', self._user.private_vp_id], 'repair': True, 'require_lock': True}) + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + # Now release the lock and run dbchk in repair mode again. + self._RunAsync(lock.Release, self._client) + self._RunDbChk({'viewpoints': ['vp1', self._user.private_vp_id], 'repair': True, 'require_lock': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testSmartScan(self): + """Test detection of last scan and smart-scan setting.""" + # Change viewpoint last_updated times to be more ancient. + vp1 = self._RunAsync(Viewpoint.Query, self._client, self._user.private_vp_id, None) + vp1.last_updated = time.time() - constants.SECONDS_PER_DAY + self._RunAsync(vp1.Update, self._client) + + vp2 = self._RunAsync(Viewpoint.Query, self._client, self._user2.private_vp_id, None) + vp2.last_updated = time.time() - constants.SECONDS_PER_DAY * 2 + self._RunAsync(vp2.Update, self._client) + + # Job to query/write run entries. + job = Job(self._client, 'dbchk') + prev_runs = self._RunAsync(job.FindPreviousRuns) + self.assertEqual(len(prev_runs), 0) + + # Run a partial dbchk. + self._RunDbChk({'viewpoints': [self._user.private_vp_id]}) + prev_runs = self._RunAsync(job.FindPreviousRuns, status=Job.STATUS_SUCCESS) + self.assertEqual(len(prev_runs), 1) + + # Wait a second between runs so they end up in different entries. + time.sleep(1) + + # Run a full dbchk with smart scan: previous run was a partial scan, so we proceed and scan everything. + self._RunDbChk({'viewpoints': [], 'smart_scan': True}) + prev_runs = self._RunAsync(job.FindPreviousRuns, status=Job.STATUS_SUCCESS) + self.assertEqual(len(prev_runs), 2) + + # Wait a second between runs so they end up in different entries. + time.sleep(1) + + # A failed dbchk run does not impact the following run. + # We trigger a failure by specifying a non-existing viewpoint. + self._RunDbChk({'viewpoints': ['vp1']}) + prev_runs = self._RunAsync(job.FindPreviousRuns, status=Job.STATUS_SUCCESS) + self.assertEqual(len(prev_runs), 2) + prev_runs = self._RunAsync(job.FindPreviousRuns, status=Job.STATUS_FAILURE) + self.assertEqual(len(prev_runs), 1) + + # Run smart scan again. This time it should not scan anything. + self._RunDbChk({'viewpoints': [], 'smart_scan': True}) + prev_runs = self._RunAsync(job.FindPreviousRuns, status=Job.STATUS_SUCCESS) + self.assertEqual(len(prev_runs), 3) + + # Verify per-run stats. + self.assertEqual(prev_runs[0]['stats.full_scan'], False) + self.assertEqual(prev_runs[0]['stats.dbchk.visited_viewpoints'], 1) + self.assertEqual(prev_runs[1]['stats.full_scan'], True) + self.assertEqual(prev_runs[1]['stats.dbchk.visited_viewpoints'], 2) + self.assertEqual(prev_runs[2]['stats.full_scan'], True) + self.assertEqual(prev_runs[2]['stats.dbchk.visited_viewpoints'], 0) + + + def testEmptyViewpoints(self): + """Verifies detection of empty viewpoint records.""" + def _Validate(viewpoint_id): + self._validator.ValidateDeleteDBObject(Follower, DBKey(self._user.user_id, viewpoint_id)) + self._validator.ValidateDeleteDBObject(Follower, DBKey(self._user2.user_id, viewpoint_id)) + + sort_key = Followed.CreateSortKey(viewpoint_id, 0) + self._validator.ValidateDeleteDBObject(Followed, DBKey(self._user.user_id, sort_key)) + self._validator.ValidateDeleteDBObject(Followed, DBKey(self._user2.user_id, sort_key)) + + self._validator.ValidateDeleteDBObject(Viewpoint, viewpoint_id) + + # Create empty non-default viewpoints (already have empty default viewpoints). + self._CreateTestViewpoint('vp1', self._user.user_id, []) + self._CreateTestViewpoint('vp2', self._user.user_id, []) + + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1,vp2' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1', 'vp2', self._user.private_vp_id], 'repair': True}) + _Validate('vp1') + _Validate('vp2') + + def testInvalidViewpointMetadata(self): + """Verifies detection of invalid viewpoint metadata.""" + viewpoint = self._CreateTestViewpoint('vp1', self._user.user_id, []) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a1', + time.time() - 10, 0, [{'new_episode_id': 'ep1', 'photo_ids': []}], []) + + # Bypass checks in DBObject and force last_updated and timestamp to be updated to None. + viewpoint = Viewpoint.CreateFromKeywords(viewpoint_id='vp1') + viewpoint._columns['last_updated'].SetModified(True) + viewpoint._columns['timestamp'].SetModified(True) + self._RunAsync(viewpoint.Update, self._client) + + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' invalid viewpoint metadata (2 instances)\n' \ + ' missing followed (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testMissingFollowed(self): + """Verifies detection and repair of missing Followed records.""" + def _Validate(follower_id, viewpoint_id, last_updated): + sort_key = Followed.CreateSortKey(viewpoint_id, last_updated) + self._validator.ValidateCreateDBObject(Followed, + user_id=follower_id, + sort_key=sort_key, + date_updated=Followed._TruncateToDay(last_updated), + viewpoint_id=viewpoint_id) + + invalidate = NotificationManager._CreateViewpointInvalidation(viewpoint_id) + self._validator.ValidateNotification('dbchk add_followed', + follower_id, + self._op_dict, + invalidate) + + # Remove Followed record from default viewpoint. + sort_key = Followed.CreateSortKey(self._user.private_vp_id, util._TEST_TIME) + followed = self._RunAsync(Followed.Query, self._client, self._user.user_id, sort_key, None) + self._RunAsync(followed.Delete, self._client) + + # Create non-default viewpoint and change last_updated to "orphan" followed records. + viewpoint = self._CreateTestViewpoint('vp1', self._user.user_id, [self._user2.user_id]) + viewpoint.last_updated += constants.SECONDS_PER_DAY + self._RunAsync(viewpoint.Update, self._client) + self._RunAsync(Activity.CreateAddFollowers, self._client, self._user.user_id, 'vp1', 'a1', + time.time(), 0, [self._user2.user_id]) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Default viewpoints created by DBBaseTestCase are missing Followed records. + corruption_text = \ + ' ---- viewpoint v-F- ----\n' \ + ' missing followed (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp1 ----\n' \ + ' missing followed (2 instances)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=v-F-,vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1', 'v-F-'], 'repair': True}) + _Validate(self._user.user_id, 'vp1', util._TEST_TIME) + _Validate(self._user.user_id, 'v-F-', util._TEST_TIME) + + def testMultipleShareNew(self): + """Verifies detection of multiple share_new activities.""" + self._CreateTestViewpoint('vp1', self._user.user_id, [self._user2.user_id]) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a1', + time.time() + 1, 0, [{'new_episode_id': 'ep2', 'photo_ids': []}], [self._user2.user_id]) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a2', + time.time(), 0, [{'new_episode_id': 'ep1', 'photo_ids': []}], [self._user2.user_id]) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a3', + time.time() + 2, 0, [{'new_episode_id': 'ep3', 'photo_ids': []}], []) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a4', + time.time() + 3, 0, [{'new_episode_id': 'ep4', 'photo_ids': []}], [self._user.user_id]) + self._RunAsync(Activity.CreateShareNew, self._client, self._user2.user_id, 'vp1', 'a5', + time.time() - 1, 0, [{'new_episode_id': 'ep5', 'photo_ids': []}], [self._user.user_id]) + + self._RunAsync(self._checker.CheckAllViewpoints) + + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' multiple share_new activities (4 instances)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testMissingActivities(self): + """Verifies detection of missing activities.""" + self._CreateTestViewpoint('vp1', self._user.user_id, [self._user2.user_id]) + self._CreateTestEpisode('vp1', 'ep1', self._user.user_id) + self._CreateTestEpisode('vp1', 'ep2', self._user.user_id) + self._CreateTestEpisode('vp1', 'ep3', self._user.user_id) + self._CreateTestViewpoint('vp2', self._user.user_id, [self._user2.user_id]) + self._CreateTestEpisode('vp2', 'ep4', self._user.user_id) + self._CreateTestComment('vp2', 'cm1', self._user.user_id, 'a comment') + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp2', 'a1', + time.time(), 0, [{'new_episode_id': 'ep4', 'photo_ids': []}], []) + self._CreateTestEpisode(self._user.private_vp_id, 'ep5', self._user.user_id) + + # Create an episode that would be created by save_photos. + ep_dict = {'episode_id': 'ep6', + 'user_id': self._user.user_id, + 'viewpoint_id': self._user.private_vp_id, + 'parent_ep_id': 'ep4', + 'publish_timestamp': time.time(), + 'timestamp': time.time()} + self._RunAsync(Episode.CreateNew, self._client, **ep_dict) + + # Create viewpoint that's missing a follower activity. + self._CreateTestViewpoint('vp3', self._user.user_id, []) + follower = Follower.CreateFromKeywords(user_id=self._user2.user_id, viewpoint_id='vp3') + self._RunAsync(follower.Update, self._client) + + # Create viewpoint with a follower covered by merge_accounts (shouldn't be tagged as missing follower). + self._CreateTestViewpoint('vp4', self._user.user_id, [self._user2.user_id]) + self._RunAsync(Activity.CreateMergeAccounts, self._client, self._user.user_id, 'vp2', 'a1', + time.time(), 0, self._user2.user_id, self._user.user_id) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Default viewpoints created by DBBaseTestCase are missing Followed records. + corruption_text = \ + ' ---- viewpoint v-F- ----\n' \ + ' missing save_photos activity (1 instance)\n' \ + ' missing upload_episode activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp4 ----\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp1 ----\n' \ + ' missing share_existing activity (2 instances)\n' \ + ' missing share_new activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp3 ----\n' \ + ' missing followed (1 instance)\n' \ + ' empty viewpoint (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' missing post_comment activity (1 instance)\n' \ + ' missing add_followers activity (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=v-F-,vp4,vp1,vp3,vp2' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1', 'vp2', 'vp3', 'vp4', 'v-F-'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testMissingSomePosts(self): + """Verifies detection of activities that refer to some missing posts.""" + # Create activity that has one episode referring to two posts where only one exists. + self._CreateTestViewpoint('vp1', self._user.user_id, []) + self._CreateTestEpisode('vp1', 'ep1', self._user.user_id) + # Skip creation of one of the posts so that a repair is needed. + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, {'photo_id':'p10'}) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a1', + time.time() + 1, 0, [{'new_episode_id': 'ep1', 'photo_ids': ['p10', 'p11']}], [self._user2.user_id]) + + # Now, create something with more complexity where only a few of the activities/episodes need repair. + # There are 2 problems with this viewpoint: + # * Activity a2, episode ep2 is missing photo p21 + # * Activity a4, episode ep5 is missing photo p50 + self._CreateTestViewpoint('vp2', self._user.user_id, []) + self._CreateTestEpisode('vp2', 'ep2', self._user.user_id) + # Create a post, but skip the other one to cause a corruption. + self._CreateTestPhotoAndPosts('ep2', self._user.user_id, {'photo_id':'p20'}) + # Create another episode with posts. + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, {'photo_id':'p30'}) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, {'photo_id':'p31'}) + self._CreateTestEpisode('vp2', 'ep3', self._user.user_id) + self._RunAsync(Activity.CreateShareNew, + self._client, + self._user.user_id, + 'vp2', + 'a2', + time.time() + 2, + 0, + [{'new_episode_id': 'ep2', 'photo_ids': ['p20', 'p21']}, + {'new_episode_id': 'ep3', 'photo_ids': ['p30', 'p31']}], + [self._user2.user_id]) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, {'photo_id':'p40'}) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, {'photo_id':'p41'}) + self._CreateTestEpisode('vp2', 'ep4', self._user.user_id) + self._RunAsync(Activity.CreateShareExisting, + self._client, + self._user.user_id, + 'vp2', + 'a3', + time.time() + 3, + 0, + [{'new_episode_id': 'ep4', 'photo_ids': ['p40', 'p41']}]) + # Skip creation of one of the posts to require a repair. + self._CreateTestPhotoAndPosts('ep5', self._user.user_id, {'photo_id':'p51'}) + self._CreateTestEpisode('vp2', 'ep5', self._user.user_id) + self._RunAsync(Activity.CreateShareExisting, + self._client, + self._user.user_id, + 'vp2', + 'a4', + time.time() + 4, + 0, + [{'new_episode_id': 'ep5', 'photo_ids': ['p50', 'p51']}]) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Viewpoint is missing post in an activity. + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' missing post referenced by activity (1 instance)\n' \ + ' viewpoint cover_photo is set to None and there are qualified photos available (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' missing post referenced by activity (1 instance)\n' \ + ' viewpoint cover_photo is set to None and there are qualified photos available (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1,vp2' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1', 'vp2'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testMissingAllPostsFromEpisode(self): + """Verifies detection of activities that refer to all missing posts from an episode.""" + # Create activity that has one episode referring to two posts, neither of which exist. + self._CreateTestViewpoint('vp1', self._user.user_id, []) + self._CreateTestEpisode('vp1', 'ep1', self._user.user_id) + # Don't create any posts for this. + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a1', + time.time() + 1, 0, [{'new_episode_id': 'ep1', 'photo_ids': ['p10', 'p11']}], [self._user2.user_id]) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Viewpoint is missing posts in an activity. + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' no posts found for episode referenced by activity (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + # Is *not* None because we don't fix this particular case, right now. + self.assertIsNotNone(self._checker._email_args) + + def testBadViewpointAccounting(self): + """Verifies detection and repair of bad accounting entries at the viewpoint level. + We also test user-level OWNED_BY since it's a 1-1 mapping with default viewpoint:OWNED_BY. + """ + # Accounting entries to write. + accounting = {} + + # Photos created. Some may be labeled as removed or unshared. + # Some photos may be missing some or all size fields. + ph_dicts = [ {'photo_id':'p0', 'tn_size':1, 'med_size':10, 'full_size':100, 'orig_size':1000}, + {'photo_id':'p1', 'tn_size':2, 'med_size':20, 'full_size':200, 'orig_size':2000}, + {'photo_id':'p2', 'tn_size':4, 'med_size':40, 'full_size':400, 'orig_size':4000}, + {'photo_id':'p3', 'tn_size':8, 'med_size':80, 'full_size':800, 'orig_size':8000}, + {'photo_id':'p4', 'tn_size':16, 'med_size':160, 'full_size':1600, 'orig_size':16000}, + {'photo_id':'p5', 'tn_size':32, 'orig_size':32000} ] + + # Add unshared and removed photos to the default viewpoint. Unshared + # photos are still counted towards accounting. + # Accounting is correct for this viewpoint. + act_ob = Accounting.CreateViewpointOwnedBy(self._user.private_vp_id, self._user.user_id) + self._CreateTestEpisode(self._user.private_vp_id, 'ep1', self._user.user_id) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[0], unshared=True) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[1], removed=True) + self._CreateTestAccounting(act_ob) + + user_ob = Accounting.CreateUserOwnedBy(self._user.user_id) + user_ob.CopyStatsFrom(act_ob) + self._CreateTestAccounting(user_ob) + + # Add unshared and removed photos to the default viewpoint for user 2. + # Wrong viewpoint-level accounting, and missing user-level accounting. + act_ob = Accounting.CreateViewpointOwnedBy(self._user2.private_vp_id, self._user2.user_id) + self._CreateTestEpisode(self._user2.private_vp_id, 'ep2.1', self._user2.user_id) + self._CreateTestPhotoAndPosts('ep2.1', self._user2.user_id, ph_dicts[0], unshared=True) + self._CreateTestPhotoAndPosts('ep2.1', self._user2.user_id, ph_dicts[1], removed=True) + self._CreateTestPhotoAndPosts('ep2.1', self._user2.user_id, ph_dicts[2]) + act_ob.IncrementFromPhotoDicts([ph_dicts[1]]) + self._CreateTestAccounting(act_ob) + + self._CreateTestViewpoint('vp1', self._user.user_id, [self._user2.user_id]) + # No photos in episode. No entries will be created for user2. + self._CreateTestEpisode('vp1', 'ep2', self._user2.user_id) + + # Some photos, including unshared. + act_sb = Accounting.CreateViewpointSharedBy('vp1', self._user.user_id) + act_vt = Accounting.CreateViewpointVisibleTo('vp1') + act_user_sb = Accounting.CreateUserSharedBy(self._user.user_id) + act_user_vt = Accounting.CreateUserVisibleTo(self._user.user_id) + self._CreateTestEpisode('vp1', 'ep3', self._user.user_id) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[1]) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[2], unshared=True) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[3], unshared=True) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[4]) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[5]) + self._SetCoverPhotoOnViewpoint('vp1', 'ep3', ph_dicts[0]['photo_id']) + + act_sb.IncrementFromPhotoDicts(ph_dicts) + act_sb.DecrementFromPhotoDicts([ph_dicts[2], ph_dicts[3]]) + self._CreateTestAccounting(act_sb) + act_user_sb.CopyStatsFrom(act_sb) + self._CreateTestAccounting(act_user_sb) + + act_vt.IncrementFromPhotoDicts(ph_dicts) + act_vt.DecrementFromPhotoDicts([ph_dicts[2], ph_dicts[3]]) + self._CreateTestAccounting(act_vt) + act_user_vt.CopyStatsFrom(act_vt) + self._CreateTestAccounting(act_user_vt) + + # No accounting entry in table for either viewpoint or user. + # User-level accounting depends on correct (or fixed) viewpoint-level accounting, + # so the missing user entries for vp2 will not show up. + self._CreateTestViewpoint('vp2', self._user.user_id, [self._user2.user_id]) + self._CreateTestEpisode('vp2', 'ep4', self._user.user_id) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, ph_dicts[1]) + self._SetCoverPhotoOnViewpoint('vp2', 'ep4', ph_dicts[0]['photo_id']) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Default viewpoints created by DBBaseTestCase are missing Followed records. + corruption_text = \ + ' ---- viewpoint v-F- ----\n' \ + ' missing upload_episode activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint v-V- ----\n' \ + ' missing accounting (1 instance)\n' \ + ' wrong accounting (1 instance)\n' \ + ' missing upload_episode activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp1 ----\n' \ + ' missing share_existing activity (1 instance)\n' \ + ' missing share_new activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' missing share_new activity (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=v-F-,v-V-,vp1,vp2' + + print self._checker._email_args['text'] + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['v-F-', 'v-V-', 'vp1', 'vp2' ], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testBadUserAccounting(self): + """Verifies detection and repair of bad accounting entries at the user level.""" + + # Photos created. Some may be labeled as removed or unshared. + # Some photos may be missing some or all size fields. + ph_dicts = [ {'photo_id':'p0', 'tn_size':1, 'med_size':10, 'full_size':100, 'orig_size':1000}, + {'photo_id':'p1', 'tn_size':2, 'med_size':20, 'full_size':200, 'orig_size':2000} ] + + # Accurate viewpoint-level accounting (simulates the previous case + fixes), but + # missing user-level entries. + # User-level accounting depends on correct (or fixed) viewpoint-level accounting, + self._CreateTestViewpoint('vp1', self._user.user_id, [self._user2.user_id]) + self._CreateTestEpisode('vp1', 'ep1', self._user.user_id) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[1]) + self._SetCoverPhotoOnViewpoint('vp1', 'ep1', ph_dicts[0]['photo_id']) + act_sb = Accounting.CreateViewpointSharedBy('vp1', self._user.user_id) + act_vt = Accounting.CreateViewpointVisibleTo('vp1') + act_sb.IncrementFromPhotoDicts(ph_dicts[0:2]) + act_vt.IncrementFromPhotoDicts(ph_dicts[0:2]) + self._CreateTestAccounting(act_sb) + self._CreateTestAccounting(act_vt) + # Count only one photo for VISIBLE_2:user2 + act_user_vt = Accounting.CreateUserVisibleTo(self._user2.user_id) + act_user_vt.IncrementFromPhotoDicts(ph_dicts[0:1]) + self._CreateTestAccounting(act_user_vt) + + # We will detect the following user-level accounting problems: + # - missing SHARED_BY for self._user + # - missing VISIBLE_TO for both self._user + # - wrong VISIBLE_TO for both self._user2 + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Default viewpoints created by DBBaseTestCase are missing Followed records. + corruption_text = \ + ' ---- viewpoint vp1 ----\n' \ + ' wrong user accounting (1 instance)\n' \ + ' missing share_new activity (1 instance)\n' \ + ' missing user accounting (2 instances)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1' ], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testBadRemovedUserAccounting(self): + """Verifies detection and repair of bad accounting entries at the user level for a REMOVED follower.""" + + # Photos created. Some may be labeled as removed or unshared. + # Some photos may be missing some or all size fields. + ph_dicts = [ {'photo_id':'p0', 'tn_size':1, 'med_size':10, 'full_size':100, 'orig_size':1000}, + {'photo_id':'p1', 'tn_size':2, 'med_size':20, 'full_size':200, 'orig_size':2000} ] + + # Create accurate user and viewpoint level accounting as if no followers are removed. + self._CreateTestViewpoint('vp1', self._user.user_id, []) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp1', 'a1', + time.time(), 0, [{'new_episode_id': 'ep1', 'photo_ids': []}], []) + self._CreateTestEpisode('vp1', 'ep1', self._user.user_id) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[1]) + self._SetCoverPhotoOnViewpoint('vp1', 'ep1', ph_dicts[0]['photo_id']) + # Create correct viewpoint level accounting. + act_sb = Accounting.CreateViewpointSharedBy('vp1', self._user.user_id) + act_vt = Accounting.CreateViewpointVisibleTo('vp1') + act_sb.IncrementFromPhotoDicts(ph_dicts[0:2]) + act_vt.IncrementFromPhotoDicts(ph_dicts[0:2]) + self._CreateTestAccounting(act_sb) + self._CreateTestAccounting(act_vt) + # Create correct user level accounting for self._user. + act_user_vt = Accounting.CreateUserVisibleTo(self._user.user_id) + act_user_vt.IncrementFromPhotoDicts(ph_dicts[0:2]) + self._CreateTestAccounting(act_user_vt) + act_user_sb = Accounting.CreateUserSharedBy(self._user.user_id) + act_user_sb.IncrementFromPhotoDicts(ph_dicts[0:2]) + self._CreateTestAccounting(act_user_sb) + + # Set the REMOVED label on the follower to see that we correctly identify the incorrect accounting. + follower = self._RunAsync(Follower.Query, self._client, self._user.user_id, 'vp1', None) + follower.labels.add(Follower.REMOVED) + self._RunAsync(follower.Update, self._client) + + # We will detect the following user-level accounting problems: + # - wrong VISIBLE_TO and SHARED_BY for self._user + + self._RunAsync(self._checker.CheckAllViewpoints) + + # User accounting is wrong for both visible_to and shared_by records. + corruption_text = \ + ' ---- viewpoint vp1 ----\n'\ + ' wrong user accounting (2 instances)\n'\ + '\n'\ + 'python dbchk.py --devbox --repair=True --viewpoints=vp1' + + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['vp1' ], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def testBadCoverPhoto(self): + """Test various corruption code paths for bad cover_photos.""" + + # Photos created. Some may be labeled as removed or unshared. + ph_dicts = [ {'photo_id':'p0'}, + {'photo_id':'p1'}, + {'photo_id':'p2'}] + + # Should be OK to not have a cover photo set on a default viewpoint. + self._CreateTestEpisode(self._user.private_vp_id, 'ep1', self._user.user_id) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[0], unshared=True) + self._CreateTestPhotoAndPosts('ep1', self._user.user_id, ph_dicts[1], removed=True) + + # Some photos, including unshared/removed. Cover_photo already correctly set. + self._CreateTestViewpoint('vp2', self._user.user_id, []) + self._CreateTestEpisode('vp2', 'ep2', self._user.user_id) + self._CreateTestPhotoAndPosts('ep2', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep2', self._user.user_id, ph_dicts[1], unshared=True) + self._CreateTestPhotoAndPosts('ep2', self._user.user_id, ph_dicts[2], removed=True) + self._SetCoverPhotoOnViewpoint('vp2', 'ep2', ph_dicts[0]['photo_id']) + + # Corruption: Some photos, but didn't set a cover_photo. + self._CreateTestViewpoint('vp3', self._user.user_id, []) + self._CreateTestEpisode('vp3', 'ep3', self._user.user_id) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep3', self._user.user_id, ph_dicts[1]) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp3', 'a3', + time.time() + 1, 0, [{'new_episode_id': 'ep3', + 'photo_ids': ['p0', 'p1']}], []) + + # Some photos, but all are removed/unshared. Cover_photo not set. Should be OK. + self._CreateTestViewpoint('vp4', self._user.user_id, []) + self._CreateTestEpisode('vp4', 'ep4', self._user.user_id) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, ph_dicts[0], unshared=True) + self._CreateTestPhotoAndPosts('ep4', self._user.user_id, ph_dicts[1], removed=True) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp4', 'a4', + time.time(), 0, [{'new_episode_id': 'ep4', + 'photo_ids': ['p0', 'p1']}], []) + + # Corruption: Some photos, but all are removed/unshared. Cover_photo set to one of them. + self._CreateTestViewpoint('vp5', self._user.user_id, []) + self._CreateTestEpisode('vp5', 'ep5', self._user.user_id) + self._CreateTestPhotoAndPosts('ep5', self._user.user_id, ph_dicts[0], unshared=True) + self._CreateTestPhotoAndPosts('ep5', self._user.user_id, ph_dicts[1], removed=True) + self._SetCoverPhotoOnViewpoint('vp5', 'ep5', ph_dicts[1]['photo_id']) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp5', 'a5', + time.time() - 1, 0, [{'new_episode_id': 'ep5', + 'photo_ids': ['p0', 'p1']}], []) + + # Corruption: Cover_photo property in bad state. Cover_photo is not None, but missing key. + self._CreateTestViewpoint('vp6', self._user.user_id, []) + self._CreateTestEpisode('vp6', 'ep6', self._user.user_id) + self._CreateTestPhotoAndPosts('ep6', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep6', self._user.user_id, ph_dicts[1]) + viewpoint = self._RunAsync(Viewpoint.Query, self._client, 'vp6', None) + viewpoint.cover_photo = {'episode_id': 'ep6'} # intentionally omit photo_id. + self._RunAsync(viewpoint.Update, self._client) + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp6', 'a6', + time.time() - 2, 0, [{'new_episode_id': 'ep6', + 'photo_ids': ['p0', 'p1']}], []) + + # Corruption: Cover_photo referenced photo not in viewpoint. + self._CreateTestViewpoint('vp7', self._user.user_id, []) + self._CreateTestEpisode('vp7', 'ep7', self._user.user_id) + self._CreateTestPhotoAndPosts('ep7', self._user.user_id, ph_dicts[0]) + self._CreateTestPhotoAndPosts('ep7', self._user.user_id, ph_dicts[1]) + self._SetCoverPhotoOnViewpoint('vp7', 'ep7', 'p99') + self._RunAsync(Activity.CreateShareNew, self._client, self._user.user_id, 'vp7', 'a7', + time.time() - 2, 0, [{'new_episode_id': 'ep7', + 'photo_ids': ['p0', 'p1']}], []) + + self._RunAsync(self._checker.CheckAllViewpoints) + + # Default viewpoints created by DBBaseTestCase are missing Followed records. + corruption_text = \ + ' ---- viewpoint vp5 ----\n' \ + ' viewpoint cover_photo is not qualified to be a cover_photo (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp7 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' viewpoint cover_photo does not match any photo in viewpoint (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp6 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' viewpoint cover_photo is not None, but does not have proper keys (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp3 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' viewpoint cover_photo is set to None and there are qualified photos available (1 instance)\n' \ + '\n' \ + ' ---- viewpoint vp2 ----\n' \ + ' missing accounting (2 instances)\n' \ + ' missing share_new activity (1 instance)\n' \ + '\n' \ + ' ---- viewpoint v-F- ----\n' \ + ' missing upload_episode activity (1 instance)\n' \ + '\n' \ + 'python dbchk.py --devbox --repair=True --viewpoints=vp5,vp7,vp6,vp3,vp2,v-F-' + + print self._checker._email_args['text'] + self.assertEqual(self._checker._email_args['text'], + 'Found corruption(s) in database:\n\n%s' % corruption_text) + + self._RunDbChk({'viewpoints': ['v-F-', 'vp2', 'vp3', 'vp4', 'vp5', 'vp6', 'vp7'], 'repair': True}) + + # Validate by checking again and finding no issues. + self._RunAsync(self._checker.CheckAllViewpoints) + self.assertIsNone(self._checker._email_args) + + def _CreateTestViewpoint(self, viewpoint_id, user_id, follower_ids, delete_followed=False): + """Create viewpoint_id for testing purposes.""" + vp_dict = {'viewpoint_id': viewpoint_id, + 'user_id': user_id, + 'timestamp': util._TEST_TIME, + 'last_updated': util._TEST_TIME, + 'type': Viewpoint.EVENT} + viewpoint, _ = self._RunAsync(Viewpoint.CreateNewWithFollowers, self._client, follower_ids, **vp_dict) + + if delete_followed: + for f_id in [user_id] + follower_ids: + sort_key = Followed.CreateSortKey(viewpoint_id, util._TEST_TIME) + followed = self._RunAsync(Followed.Query, self._client, f_id, sort_key, None) + self._RunAsync(followed.Delete, self._client) + + return viewpoint + + def _CreateTestEpisode(self, viewpoint_id, episode_id, user_id): + """Create episode for testing purposes.""" + ep_dict = {'episode_id': episode_id, + 'user_id': user_id, + 'viewpoint_id': viewpoint_id, + 'publish_timestamp': time.time(), + 'timestamp': time.time()} + return self._RunAsync(Episode.CreateNew, self._client, **ep_dict) + + def _CreateTestComment(self, viewpoint_id, comment_id, user_id, message): + """Create comment for testing purposes.""" + comment = Comment.CreateFromKeywords(viewpoint_id=viewpoint_id, comment_id=comment_id, + user_id=user_id, message=message) + self._RunAsync(comment.Update, self._client) + return comment + + def _CreateTestPhotoAndPosts(self, episode_id, user_id, ph_dict, unshared=False, removed=False): + """Create photo/post/user_post for testing purposes.""" + self._CreateTestPhoto(ph_dict) + self._CreateTestPost(episode_id, ph_dict['photo_id'], unshared=unshared, removed=removed) + + def _CreateTestPost(self, episode_id, photo_id, unshared=False, removed=False): + """Create post for testing purposes.""" + post = Post.CreateFromKeywords(episode_id=episode_id, photo_id=photo_id) + if unshared: + post.labels.add(Post.UNSHARED) + if unshared or removed: + post.labels.add(Post.REMOVED) + self._RunAsync(post.Update, self._client) + + def _CreateTestPhoto(self, ph_dict): + """Create photo for testing purposes.""" + photo = Photo.CreateFromKeywords(**ph_dict) + self._RunAsync(photo.Update, self._client) + + def _CreateTestAccounting(self, act): + """Create accounting entry for testing purposes.""" + self._RunAsync(act.Update, self._client) + + def _SetCoverPhotoOnViewpoint(self, viewpoint_id, episode_id, photo_id): + """Updates a viewpoint with the given selected cover_photo.""" + viewpoint = self._RunAsync(Viewpoint.Query, self._client, viewpoint_id, None) + viewpoint.cover_photo = Viewpoint.ConstructCoverPhoto(episode_id, photo_id) + self._RunAsync(viewpoint.Update, self._client) + + def _RunDbChk(self, option_dict=None): + """Call dbchk.Dispatch after setting the specified options.""" + if option_dict: + [setattr(options.options, name, value) for name, value in option_dict.iteritems()] + + self._RunAsync(dbchk.Dispatch, self._client) diff --git a/backend/db/test/device_test.py b/backend/db/test/device_test.py new file mode 100644 index 0000000..9b65b7b --- /dev/null +++ b/backend/db/test/device_test.py @@ -0,0 +1,67 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for device object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import time +import unittest + +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.device import Device + +from base_test import DBBaseTestCase + +class DeviceTestCase(DBBaseTestCase): + @async_test + def testInvalidToken(self): + """Create a device with an invalid token and verify it is cleared + on a push notification. + """ + self._mobile_dev.push_token = 'invalid-scheme:push-token' + self._mobile_dev.Update(self._client, self._OnDeviceUpdate) + + def _OnDeviceUpdate(self): + """Try pushing a notification to the device.""" + Device.PushNotification(self._client, self._user.user_id, 'test alert', 1, + partial(self._QueryUntilPushTokenNone, 0, self._mobile_dev)) + + def _QueryUntilPushTokenNone(self, count, device): + """Query the device until the push token has been cleared.""" + MAX_RETRIES = 5 + assert count < MAX_RETRIES + if device.push_token is None: + self.assertIsNone(device.alert_user_id) + self.stop() + else: + query_cb = partial(Device.Query, self._client, self._user.user_id, + self._mobile_dev.device_id, None, + partial(self._QueryUntilPushTokenNone, count + 1)) + self.io_loop.add_timeout(time.time() + 0.100, query_cb) + + def testRepr(self): + device = Device.CreateFromKeywords(user_id=1, device_id=1, os='iOS 6.0', name='My iPhone') + self.assertIn('iOS 6.0', repr(device)) + self.assertNotIn('My iPhone', repr(device)) + + def testCreate(self): + self.assertRaises(KeyError, Device.CreateFromKeywords, user_id=1, device_id=2, device_uuid='foo') + self.assertRaises(KeyError, Device.CreateFromKeywords, user_id=1, device_id=2, test_udid='bar') + + device = Device.Create(user_id=1, device_id=1, os='iOS 6.0', name='My iPhone', device_uuid='foo', test_udid='bar') + self.assertIn('iOS 6.0', repr(device)) + self.assertNotIn('My iPhone', repr(device)) + self.assertNotIn('device_uuid', repr(device)) + self.assertNotIn('test_udid', repr(device)) + + self.assertRaises(KeyError, device.UpdateFromKeywords, user_id=1, device_id=1, + os='iOS 6.1', device_uuid='foo', test_udid='bar') + device.UpdateFields(user_id=1, device_id=1, os='iOS 6.1', device_uuid='foo') + self.assertIn('iOS 6.1', repr(device)) + self.assertNotIn('My iPhone', repr(device)) + self.assertNotIn('device_uuid', repr(device)) + self.assertNotIn('test_udid', repr(device)) diff --git a/backend/db/test/dynamodb_client_test.py b/backend/db/test/dynamodb_client_test.py new file mode 100644 index 0000000..c6fc636 --- /dev/null +++ b/backend/db/test/dynamodb_client_test.py @@ -0,0 +1,447 @@ +# -*- coding: utf-8 -*- +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for DynamoDB client. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import os +import unittest + +from boto.exception import DynamoDBResponseError +from functools import partial +from tornado import options +from viewfinder.backend.base.testing import async_test_timeout +from viewfinder.backend.base import base_options # imported for option definitions +from viewfinder.backend.base import secrets, util, counters +from viewfinder.backend.db.db_client import DBKey, UpdateAttr, RangeOperator, BatchGetRequest, DBKeySchema +from viewfinder.backend.db import dynamodb_client, vf_schema + +from base_test import DBBaseTestCase + +_table = vf_schema.SCHEMA.GetTable(vf_schema.TEST_RENAME) + + +def setupModule(): + # The existence of a setupModule method tells nose's multiprocess test runner to serialize + # the tests in this file. + pass + +@unittest.skip("needs aws credentials") +@unittest.skipIf('NO_NETWORK' in os.environ, 'no network') +class DynamoDBClientTestCase(DBBaseTestCase): + def setUp(self): + """Clears all rows from the 'Test' DynamoDB table.""" + super(DynamoDBClientTestCase, self).setUp() + options.options.domain = 'goviewfinder.com' + secrets.InitSecretsForTest() + self._client = dynamodb_client.DynamoDBClient(schema=vf_schema.SCHEMA) + self._ClearTestTable(self.stop) + self.wait(timeout=30) + + def tearDown(self): + dynamodb_client.DynamoDBClient._MAX_BATCH_SIZE = 100 + self._ClearTestTable(self.stop) + self.wait(timeout=30) + super(DynamoDBClientTestCase, self).tearDown() + + @async_test_timeout(timeout=30) + def testPutAndGet(self): + """Put an item and verify get.""" + exp_attrs = {u'a0': 2 ** 64 + 1, u'a1': 1354137666.996147, u'a2': u'test valueà朋'} + + def _VerifyGet(barrier_cb, result): + self.assertEqual(exp_attrs, result.attributes) + self.assertEqual(result.read_units, 0.5) + barrier_cb() + + def _VerifyConsistentGet(barrier_cb, result): + self.assertEqual(exp_attrs, result.attributes) + self.assertEqual(result.read_units, 1) + barrier_cb() + + def _VerifyEmptyGet(barrier_cb, result): + self.assertTrue(result is None) + barrier_cb() + + def _OnPut(result): + key = DBKey(u'1à朋', 1) + attrs = [u'a0', u'a1', u'a2'] + with util.Barrier(self.stop) as b: + self._client.GetItem(table=_table.name, key=key, + callback=partial(_VerifyGet, b.Callback()), + attributes=attrs, must_exist=False, consistent_read=False) + self._client.GetItem(table=_table.name, key=key, + callback=partial(_VerifyConsistentGet, b.Callback()), + attributes=attrs, must_exist=False, consistent_read=True) + self._client.GetItem(table=_table.name, key=DBKey('2', 2), + callback=partial(_VerifyEmptyGet, b.Callback()), + attributes=attrs, must_exist=False, consistent_read=False) + + self._client.PutItem(table=_table.name, key=DBKey(u'1à朋', 1), callback=_OnPut, + attributes={'a0': 2 ** 64 + 1, 'a1': 1354137666.996147, 'a2': 'test valueà朋'}) + + @async_test_timeout(timeout=30) + def testPutValues(self): + """Verify operation with various attribute values.""" + def _VerifyGet(result): + self.assertEqual({u'a1': 0, u'a2': u'str', + u'a3': set([0]), u'a4': set([u'strà朋'])}, result.attributes) + self.assertEqual(result.read_units, 0.5) + self.stop() + + def _OnPut(result): + self._client.GetItem(table=_table.name, key=DBKey(u'1', 1), + callback=_VerifyGet, attributes=[u'a1', u'a2', u'a3', u'a4']) + + self._client.PutItem(table=_table.name, key=DBKey(u'1', 1), callback=_OnPut, + attributes={'a1': 0, 'a2': u'str', 'a3': set([0]), + 'a4': set(['strà朋'])}) + + @async_test_timeout(timeout=30) + def testUpdate(self): + """Update an item multiple times, varying update actions and return_values.""" + def _OnFourthUpdate(result): + self.assertEquals(result.write_units, 1) + self.assertEquals(result.return_values, {u'thk': u'2', u'trk': 2, + u'a1': 10, u'a2': 'update str 2', + u'a3': set([1, 2, 3, 4, 5, 6])}) + self.stop() + + def _OnThirdUpdate(result): + self.assertEquals(result.write_units, 1) + self.assertEquals(result.return_values, {u'thk': u'2', u'trk': 2, + u'a1': 10, u'a2': 'update str 2', + u'a3': set([1, 2, 3, 4, 5, 6])}) + + # Delete non-existent value in non-existent attribute. + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnFourthUpdate, + attributes={'a4': UpdateAttr(set(['100']), 'DELETE')}, + return_values='ALL_NEW') + + def _OnSecondUpdate(result): + self.assertEquals(result.write_units, 1) + self.assertEquals(result.return_values, {u'a1': 10, u'a2': 'update str 2', + u'a3': set([1, 2, 3, 4, 5, 6]), + u'a4': set([u'3'])}) + + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnThirdUpdate, + attributes={'a10': UpdateAttr(None, 'DELETE'), + 'a4': UpdateAttr(set(['3']), 'DELETE')}, + return_values='ALL_NEW') + + def _OnFirstUpdate(result): + self.assertEquals(result.write_units, 1) + self.assertEquals(result.return_values, {u'thk': u'2', u'trk': 2, + u'a1': 5, u'a2': u'update str', + u'a3': set([1, 2, 3]), u'a4': set([u'1', u'3', u'2'])}) + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnSecondUpdate, + attributes={'a1': UpdateAttr(5, 'ADD'), + 'a2': UpdateAttr('update str 2', 'PUT'), + 'a3': UpdateAttr(set([4, 5, 6]), 'ADD'), + 'a4': UpdateAttr(set(['1', '2', '100']), 'DELETE')}, + return_values='UPDATED_NEW') + + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnFirstUpdate, + attributes={'a1': UpdateAttr(5, 'ADD'), + 'a2': UpdateAttr('update str', 'PUT'), + 'a3': UpdateAttr(set([1, 2, 3]), 'ADD'), + 'a4': UpdateAttr(set(['1', '2', '3']), 'PUT')}, + return_values='ALL_NEW') + + @async_test_timeout(timeout=30) + def testUpdateNoAttributes(self): + """Update an item with no attributes set, other than the key. This should result in a false-positive + from dynamodb that the record was updated, even though the record is not actually created. + """ + def _VerifyGet(result): + self.assertFalse(result == True) + self.stop() + + def _OnUpdate(result): + self.assertEquals(result.write_units, 1) + self._client.GetItem(table=_table.name, key=DBKey(u'2', 2), attributes=[u'a1', u'a2'], + must_exist=False, callback=_VerifyGet) + + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnUpdate, + attributes={}) + + @async_test_timeout(timeout=30) + def testUpdateWithDelete(self): + """Update an item by deleting its attributes.""" + def _OnDeleteUpdate(result): + self.assertEquals(result.write_units, 1) + self.assertEquals(result.return_values, {u'a0': 1, u'thk': u'2', u'trk': 2}) + self.stop() + + def _OnUpdate(result): + self.assertEquals(result.write_units, 1) + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnDeleteUpdate, + attributes={'a1': UpdateAttr(None, 'DELETE'), + 'a2': UpdateAttr(None, 'DELETE'), + 'a3': UpdateAttr(None, 'DELETE'), + 'a4': UpdateAttr(None, 'DELETE')}, + return_values='ALL_NEW') + + self._client.UpdateItem(table=_table.name, key=DBKey(u'2', 2), callback=_OnUpdate, + attributes={'a0': UpdateAttr(1, 'PUT'), + 'a1': UpdateAttr(5, 'PUT'), + 'a2': UpdateAttr('update str', 'PUT'), + 'a3': UpdateAttr(set([1, 2, 3]), 'PUT'), + 'a4': UpdateAttr(set(['1', '2', '3']), 'PUT')}) + + @async_test_timeout(timeout=30) + def testQuery(self): + """Adds a range of values and queries with start key and limit.""" + num_items = 10 + + def _OnQuery(exp_count, result): + for i in xrange(exp_count): + self.assertEqual(len(result.items), exp_count) + self.assertEqual(result.items[i], {u'thk': u'test_query', u'trk': i, + u'a1': i, u'a2': ('test-%d' % i)}) + self.stop() + + def _OnPutItems(): + self._client.Query(table=_table.name, hash_key='test_query', + range_operator=RangeOperator([0, 5], 'BETWEEN'), + callback=partial(_OnQuery, 6), attributes=['thk', 'trk', 'a1', 'a2']) + + with util.Barrier(_OnPutItems) as b: + for i in xrange(num_items): + self._client.PutItem(table=_table.name, key=DBKey(u'test_query', i), + callback=b.Callback(), attributes={'a1': i, 'a2': ('test-%d' % i)}) + + def _ClearTestTable(self, callback): + """Clears the contents of the 'Test' table by scanning the rows and deleting + items by composite key. + """ + def _OnScan(result): + with util.Barrier(callback) as b: + for item in result.items: + self._client.DeleteItem(table=_table.name, key=DBKey(item['thk'], item['trk']), + callback=b.Callback()) + + self._client.Scan(table=_table.name, callback=_OnScan, attributes=['thk', 'trk']) + + @async_test_timeout(timeout=30) + def testBadRequest(self): + """Verify exceptions are propagated on a bad request.""" + def _OnPut(result): + assert False, 'Put should fail' + + def _OnError(type, value, callback): + self.assertEqual(type, DynamoDBResponseError) + self.stop() + + with util.Barrier(_OnPut, _OnError) as b: + # Put an item with a blank string, which is disallowed by DynamoDB. + self._client.PutItem(table=_table.name, key=DBKey(u'1', 1), attributes={'a2': ''}, + callback=b.Callback()) + + @async_test_timeout(timeout=30) + def testExceptionPropagationInStackContext(self): + """Verify exceptions propagated from the DynamoDB client are raised in + the stack context of the caller. + """ + entered = [False, False] + + def _OnPut1(): + assert False, 'Put1 should fail' + + def _OnError1(type, value, tb): + #logging.info('in Put1 error handler') + if entered[0]: + print 'in error1 again!' + assert not entered[0], 'already entered error 1' + entered[0] = True + if all(entered): + self.stop() + + def _OnPut2(): + assert False, 'Put2 should fail' + + def _OnError2(type, value, tb): + #logging.info('in Put2 error handler') + assert not entered[1], 'already entered error 2' + entered[1] = True + if all(entered): + self.stop() + + # Pause all processing to allow two put operations to queue and for the + # latter's error handler to become the de-facto stack context. + self._client._scheduler._Pause() + + with util.Barrier(_OnPut1, _OnError1) as b1: + # Put an item with a blank string, which is disallowed by DynamoDB. + self._client.PutItem(table=_table.name, key=DBKey(u'1', 1), attributes={'a2': ''}, + callback=b1.Callback()) + + with util.Barrier(_OnPut2, _OnError2) as b2: + # Put a valid item; this should replace the previous stack context. + self._client.PutItem(table=_table.name, key=DBKey(u'2', 1), attributes={'a2': ''}, + callback=b2.Callback()) + + # Resume the request scheduler queue processing. + self._client._scheduler._Resume() + + @async_test_timeout(timeout=30) + def testPerformanceCounters(self): + """Verify that performance counters are working correctly for DynamoDB.""" + meter = counters.Meter(counters.counters.viewfinder.dynamodb) + def _PutComplete(): + self.stop() + + self._client._scheduler._Pause() + with util.Barrier(_PutComplete) as b: + self._client.PutItem(table=_table.name, key=DBKey(u'1', 1), + attributes={'a1': 100, 'a2': 'test value'}, callback=b.Callback()) + self._client.PutItem(table=_table.name, key=DBKey(u'2', 1), + attributes={'a1': 200, 'a2': 'test value'}, callback=b.Callback()) + + sample = meter.sample() + self.assertEqual(2, sample.viewfinder.dynamodb.requests_queued) + + self._client._scheduler._Resume() + sample = meter.sample() + self.assertEqual(0, sample.viewfinder.dynamodb.requests_queued) + + @async_test_timeout(timeout=30) + def testBatchGetItem(self): + """Put items and verify getting them in a batch.""" + attrs = {'a0': 123.456, 'a2': 'test value', 'a4': set(['foo', 'bar'])} + attrs2 = {'a0': 1, 'a1':-12345678901234567890, 'a3': set([1, 2, 3])} + + with util.Barrier(self.stop) as b: + self._client.PutItem(table=_table.name, key=DBKey(u'1', 1), attributes=attrs, callback=b.Callback()) + self._client.PutItem(table=_table.name, key=DBKey('test_key', 2), attributes=attrs2, callback=b.Callback()) + self.wait() + + batch_dict = {_table.name: BatchGetRequest(keys=[DBKey('1', 1), + DBKey('unknown', 0), + DBKey('test_key', 2), + DBKey('test_key', 2)], + attributes=['thk', 'trk', 'a0', 'a1', 'a2', 'a3', 'a4'], + consistent_read=True)} + + # Simple batch call. + response = self._RunAsync(self._client.BatchGetItem, batch_dict, must_exist=False) + self.assertEqual(len(response.keys()), 1) + response = response[_table.name] + + attrs.update({'thk': '1', 'trk': 1}) + attrs2.update({'thk': 'test_key', 'trk': 2}) + + self.assertEqual(response.read_units, 3.0) + self.assertEqual(response.items[0], attrs) + self.assertIsNone(response.items[1]) + self.assertEqual(response.items[2], attrs2) + self.assertEqual(response.items[3], attrs2) + + # Multiple calls to DynamoDB (force small max batch size). + dynamodb_client.DynamoDBClient._MAX_BATCH_SIZE = 2 + response = self._RunAsync(self._client.BatchGetItem, batch_dict, must_exist=False) + response = response[_table.name] + dynamodb_client.DynamoDBClient._MAX_BATCH_SIZE = 100 + + self.assertEqual(response.read_units, 3.0) + self.assertEqual(response.items[0], attrs) + self.assertIsNone(response.items[1]) + self.assertEqual(response.items[2], attrs2) + self.assertEqual(response.items[3], attrs2) + + # ERROR: must_exist == True. + self.assertRaises(AssertionError, self._RunAsync, self._client.BatchGetItem, batch_dict) + + # Trigger unprocessed keys by querying for many keys at once. + batch_dict = {_table.name: BatchGetRequest(keys=[], + attributes=['thk', 'trk'], + consistent_read=True)} + for i in xrange(25): + batch_dict[_table.name].keys.append(DBKey('unknown %d' % i, 1)) + + response = self._RunAsync(self._client.BatchGetItem, batch_dict, must_exist=False) + response = response[_table.name] + self.assertEqual(len(response.items), 25) + [self.assertIsNone(item) for item in response.items] + + # ERROR: Only support keys from single table currently. + batch_dict = {'foo': BatchGetRequest(keys=[], attributes=[], consistent_read=False), + 'bar': BatchGetRequest(keys=[], attributes=[], consistent_read=False)} + self.assertRaises(AssertionError, self._RunAsync, self._client.BatchGetItem, batch_dict) + + self.stop() + + +@unittest.skip("needs aws credentials") +@unittest.skipIf('NO_NETWORK' in os.environ, 'no network') +class DynamoDBReadOnlyClientTestCase(DBBaseTestCase): + def setUp(self): + """Creates a read-only local client. + Manually flips the _read_only variable to populate the table, then flips it back. + """ + super(DynamoDBReadOnlyClientTestCase, self).setUp() + options.options.domain = 'goviewfinder.com' + secrets.InitSecretsForTest() + self._client = dynamodb_client.DynamoDBClient(schema=vf_schema.SCHEMA, read_only=True) + + self._client._read_only = False + self._RunAsync(self._ClearTestTable) + self._RunAsync(self._client.PutItem, table=_table.name, key=DBKey(hash_key='1', range_key=2), attributes={'a1': 1}) + self._client._read_only = True + + + def tearDown(self): + dynamodb_client.DynamoDBClient._MAX_BATCH_SIZE = 100 + self._client._read_only = False + self._RunAsync(self._ClearTestTable) + super(DynamoDBReadOnlyClientTestCase, self).tearDown() + + + def _ClearTestTable(self, callback): + """Clears the contents of the 'Test' table by scanning the rows and deleting + items by composite key. + """ + def _OnScan(result): + with util.Barrier(callback) as b: + for item in result.items: + self._client.DeleteItem(table=_table.name, key=DBKey(item['thk'], item['trk']), + callback=b.Callback()) + + self._client.Scan(table=_table.name, callback=_OnScan, attributes=['thk', 'trk']) + + def testMethods(self): + # Read-only methods: + # We don't try ListTables as this would require giving user/test access on all tables. + # self._RunAsync(self._client.ListTables) + self._RunAsync(self._client.DescribeTable, _table.name) + self._RunAsync(self._client.GetItem, table=_table.name, + key=DBKey('1', 2), attributes=['a1']) + batch_dict = {_table.name: BatchGetRequest(keys=[DBKey('1', 2)], attributes=['thk', 'trk', 'a1'], + consistent_read=True)} + self._RunAsync(self._client.BatchGetItem, batch_dict) + self._RunAsync(self._client.Query, table=_table.name, hash_key='1', range_operator=None, attributes=None) + self._RunAsync(self._client.Scan, table=_table.name, attributes=None) + + # Mutating methods: + # We may not have permission to issue some of those requests against the test dynamodb table, but we'll raise + # the 'read-only' exception first. + _hash_key_schema = DBKeySchema(name='test_hk', value_type='N') + _range_key_schema = DBKeySchema(name='test_rk', value_type='N') + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.CreateTable, _table.name, _hash_key_schema, _range_key_schema, 5, 10) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.DeleteTable, table=_table.name) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.PutItem, table=_table.name, key=DBKey(hash_key=1, range_key=2), + attributes={'num': 1}) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.DeleteItem, table=_table.name, key=DBKey(hash_key=1, range_key=2)) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.UpdateItem, table=_table.name, key=DBKey(hash_key=1, range_key=2), + attributes={'num': 1}) diff --git a/backend/db/test/episode_test.py b/backend/db/test/episode_test.py new file mode 100644 index 0000000..6db3ba6 --- /dev/null +++ b/backend/db/test/episode_test.py @@ -0,0 +1,56 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Episode data object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import time + +from viewfinder.backend.base.exceptions import PermissionError +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.viewpoint import Viewpoint +from viewfinder.backend.op.op_context import EnterOpContext + +from base_test import DBBaseTestCase + +class EpisodeTestCase(DBBaseTestCase): + def testCreateAndUpdate(self): + """Creates a episode with id pre-allocated on mobile device. Then updates the episode.""" + with EnterOpContext(Operation(1, 'o1')): + timestamp = time.time() + episode_id = Episode.ConstructEpisodeId(timestamp, self._mobile_dev.device_id, 15) + ep_dict = {'user_id': self._user.user_id, + 'episode_id': episode_id, + 'viewpoint_id': self._user.private_vp_id, + 'timestamp': time.time(), + 'publish_timestamp': time.time(), + 'description': 'yada yada this is a episode', + 'title': 'Episode #1'} + + episode = self._RunAsync(Episode.CreateNew, self._client, **ep_dict) + episode._version = None + self.assertEqual(ep_dict, episode._asdict()) + + update_dict = {'episode_id': episode.episode_id, + 'user_id': episode.user_id, + 'description': 'updated description', + 'title': 'Episode #1a'} + self._RunAsync(episode.UpdateExisting, self._client, **update_dict) + episode._version = None + ep_dict.update(update_dict) + self.assertEqual(ep_dict, episode._asdict()) + + def testAnotherViewpoint(self): + """Create an episode in a non-default viewpoint.""" + vp_dict = {'viewpoint_id': 'vp1', 'user_id': 1, 'timestamp': time.time(), 'type': Viewpoint.EVENT} + self._RunAsync(Viewpoint.CreateNew, self._client, **vp_dict) + + ep_dict = {'viewpoint_id': 'vp1', 'episode_id': 'ep1', 'user_id': 1, + 'timestamp': 100, 'publish_timestamp': 100} + + episode = self._RunAsync(Episode.CreateNew, self._client, **ep_dict) + episode._version = None + self.assertEqual(ep_dict, episode._asdict()) diff --git a/backend/db/test/follower_test.py b/backend/db/test/follower_test.py new file mode 100644 index 0000000..0b72ff7 --- /dev/null +++ b/backend/db/test/follower_test.py @@ -0,0 +1,30 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Follower data object. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import time +import unittest + +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.base.exceptions import PermissionError +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.viewpoint import Viewpoint + +from base_test import DBBaseTestCase + +class FollowerTestCase(DBBaseTestCase): + def testUpdatePermissions(self): + """Try to update permission labels to the empty set.""" + follower_dict = {'user_id': 1, + 'viewpoint_id': 'vp1', + 'labels': [Follower.ADMIN]} + follower = self.UpdateDBObject(Follower, **follower_dict) + self.assertRaises(PermissionError, follower.SetLabels, []) diff --git a/backend/db/test/friend_test.py b/backend/db/test/friend_test.py new file mode 100644 index 0000000..7ff05f3 --- /dev/null +++ b/backend/db/test/friend_test.py @@ -0,0 +1,64 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for Friend data object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import unittest +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.friend import Friend + +from base_test import DBBaseTestCase + +class FriendTestCase(DBBaseTestCase): + def testMakeFriends(self): + """Creates a bidirectional friendship between two users.""" + friend, reverse_friend = self._RunAsync(Friend.MakeFriends, + self._client, + self._user.user_id, + self._user2.user_id) + + # Check the friends returned from make friends. + self.assertEqual(friend.user_id, self._user.user_id) + self.assertEqual(friend.friend_id, self._user2.user_id) + self.assertEqual(reverse_friend.user_id, self._user2.user_id) + self.assertEqual(reverse_friend.friend_id, self._user.user_id) + + def testOneSidedFriends(self): + """Test friendships that are only recognized by one of the users.""" + # Create one-sided friendship. + self._RunAsync(Friend.MakeFriendAndUpdate, + self._client, + self._user.user_id, + {'user_id': self._user2.user_id, 'nickname': 'Slick'}) + + # Forward friend should exist. + forward_friend = self._RunAsync(Friend.Query, + self._client, + self._user.user_id, + self._user2.user_id, + None, + must_exist=False) + self.assertIsNotNone(forward_friend) + + # Reverse friend should not exist. + reverse_friend = self._RunAsync(Friend.Query, + self._client, + self._user2.user_id, + self._user.user_id, + None, + must_exist=False) + self.assertIsNone(reverse_friend) + + # MakeFriends should add the bi-directional friendship. + forward_friend, reverse_friend = self._RunAsync(Friend.MakeFriends, + self._client, + self._user.user_id, + self._user2.user_id) + self.assertIsNotNone(forward_friend) + self.assertIsNotNone(reverse_friend) diff --git a/backend/db/test/health_report_test.py b/backend/db/test/health_report_test.py new file mode 100644 index 0000000..bb55f36 --- /dev/null +++ b/backend/db/test/health_report_test.py @@ -0,0 +1,194 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Health Report system. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import json +import time +from functools import partial + +from base_test import DBBaseTestCase +from viewfinder.backend.base import counters, util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.metric import Metric, MetricInterval +from viewfinder.backend.db.health_report import HealthReport, HealthCriteria, TREND_SAMPLE_SIZE + +class HealthReportTestClass(DBBaseTestCase): + + @async_test + def testHealthReport(self): + """Verify the health report generation and retrieval.""" + + # Test constants + cluster_name = 'test' + interval = MetricInterval('testint', 60) + group_key = Metric.EncodeGroupKey(cluster_name, interval) + num_machines = 5 + num_samples = TREND_SAMPLE_SIZE + 1 + + # Outer variables + fake_time = 0 + def fake_time_func(): + return fake_time + managers = [] + criteria_list = [] + criteria_called = [0, 0] + + # Test criteria #1 + def _deltaCriteria(totals, deltas): + criteria_called[0] += 1 + alerts = [] + warnings = [] + if len(totals['machine_data'].keys()) != len(deltas['machine_data'].keys()): + alerts.append('CLUSTER') + + warnings = [m for m, v in deltas['machine_data'].iteritems() if v > 3] + return alerts, warnings + + # Test criteria #2 + def _rateCriteria(rates): + criteria_called[1] += 1 + + warnings = [] + for m, v in rates['machine_data'].iteritems(): + m_int = int(m[len('machine'):]) + if v == m_int / interval.length: + warnings.append(m) + + return [], warnings + + def _OnGetReport(repeat, report): + # Verify that the criteria were actually executed. + for i in criteria_called: + self.assertEqual(num_samples, i) + + # Verify that the criteria generated the expected warnings. + self.assertEqual(len(report.alerts), num_machines) + self.assertFalse('deltaCrit:CLUSTER' in report.alerts) + + for i in range(1, num_machines + 1): + m_name = 'machine%d' % i + self.assertTrue(('rateCrit:' + m_name) in report.warnings) + self.assertTrue(('rateCrit:' + m_name) in report.alerts) + + if i > 3: + self.assertTrue(('deltaCrit:' + m_name) in report.warnings) + else: + self.assertFalse(('deltaCrit:' + m_name) in report.warnings) + + if repeat: + # Repeat the call to GetHealthReport to verify that criteria are only + # run when the report is generated. + HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length, + partial(_OnGetReport, False), managers[0], criteria_list) + else: + self.stop() + + + def _OnMetricsUploaded(): + # Create a criteria list and request a health report based on those criteria. + criteria_list.append(HealthCriteria('deltaCrit', + 'Description', + _deltaCriteria, + [managers[0].aggtest.total, managers[0].aggtest.delta], + 0)) + criteria_list.append(HealthCriteria('rateCrit', + 'Description', + _rateCriteria, + [managers[0].aggtest.rate], + 5)) + + HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length, + partial(_OnGetReport, True), managers[0], criteria_list) + + + + with util.Barrier(_OnMetricsUploaded) as b: + # Generate metrics. + for m in range(1, num_machines + 1): + cm = counters._CounterManager() + cm.register(counters._TotalCounter('aggtest.total', 'Test Total')) + cm.register(counters._DeltaCounter('aggtest.delta', 'Test Delta')) + cm.register(counters._RateCounter('aggtest.rate', 'Test Rate', time_func=fake_time_func)) + cm.register(counters._AverageCounter('aggtest.avg', 'Test Average')) + + fake_time = 0 + meter = counters.Meter(cm) + managers.append(cm) + for s in range(num_samples): + cm.aggtest.total.increment(m) + cm.aggtest.delta.increment(m) + cm.aggtest.rate.increment(m) + cm.aggtest.avg.add(m) + cm.aggtest.avg.add(m) + fake_time += interval.length + sample = json.dumps(meter.sample()) + metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample) + metric.Update(self._client, b.Callback()) + + @async_test + def testEmptyHealthReport(self): + """Verify the health reports with no data are properly saved to the db. + """ + + # Test constants + cluster_name = 'test' + interval = MetricInterval('testint', 60) + group_key = Metric.EncodeGroupKey(cluster_name, interval) + num_machines = 5 + num_samples = TREND_SAMPLE_SIZE + 1 + + # Outer variables + managers = [] + criteria_list = [] + criteria_called = [0] + + # Test criteria #1 + def _blankCriteria(): + criteria_called[0] += 1 + return [], [] + + def _OnDirectQuery(reports): + self.assertEqual(len(reports), num_samples) + self.stop() + + def _OnGetReport(report): + # Verify that the criteria were actually executed. + for i in criteria_called: + self.assertEqual(num_samples, i) + + # Verify that the criteria generated the expected warnings. + self.assertEqual(len(report.alerts), 0) + self.assertEqual(len(report.warnings), 0) + HealthReport.QueryTimespan(self._client, group_key, interval.length, interval.length * num_samples, + _OnDirectQuery) + + + def _OnMetricsUploaded(): + # Create a criteria list and request a health report based on those criteria. + criteria_list.append(HealthCriteria('blankCrit', + 'Description', + _blankCriteria, + [], + 0)) + + HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length, + _OnGetReport, managers[0], criteria_list) + + + with util.Barrier(_OnMetricsUploaded) as b: + # Generate metrics. + for m in range(1, num_machines + 1): + cm = counters._CounterManager() + cm.register(counters._TotalCounter('aggtest.total', 'Test Total')) + + fake_time = 0 + meter = counters.Meter(cm) + managers.append(cm) + for s in range(num_samples): + fake_time += interval.length + sample = json.dumps(meter.sample()) + metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample) + metric.Update(self._client, b.Callback()) diff --git a/backend/db/test/id_allocator_test.py b/backend/db/test/id_allocator_test.py new file mode 100755 index 0000000..dd28575 --- /dev/null +++ b/backend/db/test/id_allocator_test.py @@ -0,0 +1,49 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for IdAllocator data object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import unittest + +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.id_allocator import IdAllocator + +from base_test import DBBaseTestCase + +class IdAllocatorTestCase(DBBaseTestCase): + @async_test + def testCreate(self): + alloc = IdAllocator('type', 13) + num_ids = 3000 + def _OnAllocated(ids): + id_set = set(ids) + assert len(id_set) == num_ids + self.stop() + + with util.ArrayBarrier(_OnAllocated) as b: + [alloc.NextId(self._client, callback=b.Callback()) for i in xrange(num_ids)] + + @async_test + def testMultiple(self): + """Tests that multiple allocations from the same sequence do + not overlap. + """ + allocs = [IdAllocator('type'), IdAllocator('type')] + num_ids = 3000 + def _OnAllocated(id_lists): + assert len(id_lists) == 2 + id_set1 = set(id_lists[0]) + id_set2 = set(id_lists[1]) + assert len(id_set1) == 3000 + assert len(id_set2) == 3000 + assert id_set1.isdisjoint(id_set2) + self.stop() + + with util.ArrayBarrier(_OnAllocated) as b: + with util.ArrayBarrier(b.Callback()) as b1: + [allocs[0].NextId(self._client, b1.Callback()) for i in xrange(num_ids)] + with util.ArrayBarrier(b.Callback()) as b2: + [allocs[1].NextId(self._client, b2.Callback()) for i in xrange(num_ids)] diff --git a/backend/db/test/id_test.py b/backend/db/test/id_test.py new file mode 100644 index 0000000..1777b58 --- /dev/null +++ b/backend/db/test/id_test.py @@ -0,0 +1,91 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for asset ids. + +The output of this test should be used to verify the client-side +server id generation tests. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import time +import unittest + +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.asset_id import AssetIdUniquifier +from viewfinder.backend.db.comment import Comment +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.viewpoint import Viewpoint + +class IdTestCase(unittest.TestCase): + _PRINT_RESULTS = False + + def testActivityIds(self): + """Test activity ids.""" + self._TestIdRoundTripWithTimestamp(Activity.ConstructActivityId, Activity.DeconstructActivityId) + + def testCommentIds(self): + """Test comment ids.""" + self._TestIdRoundTripWithTimestamp(Comment.ConstructCommentId, Comment.DeconstructCommentId) + + def testEpisodeIds(self): + """Test episode ids.""" + self._TestIdRoundTripWithTimestamp(Episode.ConstructEpisodeId, Episode.DeconstructEpisodeId) + + def testOperationIds(self): + """Test operation ids.""" + self._TestIdRoundTrip(Operation.ConstructOperationId, Operation.DeconstructOperationId) + + def testPhotoIds(self): + """Test photo ids.""" + self._TestIdRoundTripWithTimestamp(Photo.ConstructPhotoId, Photo.DeconstructPhotoId) + + def testViewpointIds(self): + """Test viewpoint ids.""" + self._TestIdRoundTrip(Viewpoint.ConstructViewpointId, Viewpoint.DeconstructViewpointId) + + def _TestIdRoundTripWithTimestamp(self, construct, deconstruct): + """Round-trip id with a timestamp.""" + def _RoundTrip(timestamp, device_id, uniquifier): + asset_id = construct(timestamp, device_id, uniquifier) + actual_timestamp, actual_device_id, actual_uniquifier = \ + deconstruct(asset_id) + self.assertEqual(int(timestamp), actual_timestamp) + self.assertEqual(device_id, actual_device_id) + self.assertEqual(uniquifier, actual_uniquifier) + if IdTestCase._PRINT_RESULTS: + print '%r => %s' % ((timestamp, device_id, uniquifier), asset_id) + + _RoundTrip(0, 0, (0, None)) + _RoundTrip(1234234.123423, 127, (128, None)) + _RoundTrip(time.time(), 128, (127, None)) + _RoundTrip(time.time(), 128, (128, None)) + _RoundTrip(time.time(), 123512341234, (827348273422, None)) + + _RoundTrip(0, 0, (0, 'v1234')) + _RoundTrip(1234234.123423, 127, (128, '\n\t\r\b\0abc123\x1000')) + _RoundTrip(time.time(), 128, (127, u'1')) + _RoundTrip(time.time(), 128, (128, ' ')) + + def _TestIdRoundTrip(self, construct, deconstruct): + """Round-trip id.""" + def _RoundTrip(device_id, device_local_id): + server_id = construct(device_id, device_local_id) + actual_device_id, actual_device_local_id = deconstruct(server_id) + self.assertEqual(device_id, actual_device_id) + self.assertEqual(device_local_id, actual_device_local_id) + if IdTestCase._PRINT_RESULTS: + print '%r => %s' % ((device_id, device_local_id), server_id) + + _RoundTrip(0, (0, None)) + _RoundTrip(127, (128, None)) + _RoundTrip(128, (127, None)) + _RoundTrip(128, (128, None)) + _RoundTrip(123512341234, (827348273422, None)) + + _RoundTrip(0, (0, 'v1234')) + _RoundTrip(127, (128, '\n\t\r\b\0abc123\x1000')) + _RoundTrip(128, (127, u'1')) + _RoundTrip(128, (128, ' ')) diff --git a/backend/db/test/identity_test.py b/backend/db/test/identity_test.py new file mode 100644 index 0000000..8b47a14 --- /dev/null +++ b/backend/db/test/identity_test.py @@ -0,0 +1,44 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Identity. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +from viewfinder.backend.base.exceptions import InvalidRequestError +from viewfinder.backend.db.identity import Identity + +from base_test import DBBaseTestCase + +class IdentityTestCase(DBBaseTestCase): + KEY = 'Local:test@example.com' + + def testPhoneNumbers(self): + """Test validation of phone numbers.""" + # United States. + self.assertEqual(Identity.CanonicalizePhone('+14251234567'), '+14251234567') + + # Malaysia. + self.assertEqual(Identity.CanonicalizePhone('+60321345678'), '+60321345678') + + # Great Britain. + self.assertEqual(Identity.CanonicalizePhone('+442083661177'), '+442083661177') + + # China. + self.assertEqual(Identity.CanonicalizePhone('+861082301234'), '+861082301234') + + self.assertRaises(InvalidRequestError, Identity.CanonicalizePhone, None) + self.assertRaises(InvalidRequestError, Identity.CanonicalizePhone, '') + self.assertRaises(InvalidRequestError, Identity.CanonicalizePhone, '14251234567') + self.assertRaises(InvalidRequestError, Identity.CanonicalizePhone, '+') + self.assertRaises(InvalidRequestError, Identity.CanonicalizePhone, '+abc') + + def testRepr(self): + """Test conversion of Identity objects to strings.""" + ident = Identity.CreateFromKeywords(key='Email:foo@example.com', + access_token='access_token1', + refresh_token='refresh_token1') + self.assertIn('foo@example.com', repr(ident)) + self.assertIn('scrubbed', repr(ident)) + self.assertNotIn('access_token1', repr(ident)) + self.assertNotIn('refresh_token1', repr(ident)) diff --git a/backend/db/test/indexing_test.py b/backend/db/test/indexing_test.py new file mode 100644 index 0000000..0acb095 --- /dev/null +++ b/backend/db/test/indexing_test.py @@ -0,0 +1,467 @@ +# -*- coding: utf-8 -*- +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests DB indexing and querying. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import platform +import random +import time +import unittest + +from functools import partial +from tornado import escape +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.contact import Contact +from viewfinder.backend.db.db_client import DBKey +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.schema import Location, Placemark +from viewfinder.backend.db.user import User +from viewfinder.backend.db.vf_schema import USER + +from base_test import DBBaseTestCase + +class IndexingTestCase(DBBaseTestCase): + @async_test + def testIndexing(self): + """Tests indexing of multiple objects with overlapping field values. + Creates 100 users, then queries for specific items. + """ + given_names = ['Spencer', 'Peter', 'Brian', 'Chris'] + family_names = ['Kimball', 'Mattis', 'McGinnis', 'Schoenbohm'] + emails = ['spencer.kimball@emailscrubbed.com', 'spencer@goviewfinder.com', + 'petermattis@emailscrubbed.com', 'peter.mattis@gmail.com', 'peter@goviewfinder.com', + 'brian.mcginnis@emailscrubbed.com', 'brian@goviewfinder.com', + 'chris.schoenbohm@emailscrubbed.com', 'chris@goviewfinder.com'] + + num_users = 100 + + def _QueryAndVerify(users, barrier_cb, col, value): + def _Verify(q_users): + logging.debug('querying for %s=%s yielded %d matches' % (col, value, len(q_users))) + for u in q_users: + # Exclude users created by base class. + if u.user_id not in [self._user.user_id, self._user2.user_id]: + self.assertEqual(getattr(users[u.user_id], col), value) + barrier_cb() + User.IndexQuery(self._client, ('user.%s={v}' % col, {'v': value}), + col_names=None, callback=_Verify) + + def _OnCreateUsers(user_list): + users = dict([(u.user_id, u) for u in user_list]) + with util.Barrier(self.stop) as b: + [_QueryAndVerify(users, b.Callback(), 'given_name', value) for value in given_names] + [_QueryAndVerify(users, b.Callback(), 'family_name', value) for value in family_names] + [_QueryAndVerify(users, b.Callback(), 'email', value) for value in emails] + + with util.ArrayBarrier(_OnCreateUsers) as b: + for i in xrange(num_users): + kwargs = {'user_id': i + 10, + 'given_name': random.choice(given_names), + 'family_name': random.choice(family_names), + 'email': random.choice(emails), } + user = User.CreateFromKeywords(**kwargs) + user.Update(self._client, partial(b.Callback(), user)) + + def testIndexQueryForNonExistingItem(self): + """IndexQuery should not return a result list with any None elements.""" + # Create a user: + user = User.CreateFromKeywords(user_id=1, given_name='Mike', family_name='Purtell', email='mike@time.com') + self._RunAsync(user.Update, self._client) + + # Should return one non-None item. + results = self._RunAsync(User.IndexQuery, self._client, ('user.given_name={v}', {'v': 'Mike'}), col_names=None) + self.assertEqual(len(results), 1) + self.assertIsNotNone(results[0]) + + # Delete the item that the index references. + self._RunAsync(self._client.DeleteItem, table=USER, key=user.GetKey()) + + # IndexQuery again with same query to see that a zero length list is returned. + results = self._RunAsync(User.IndexQuery, self._client, ('user.given_name={v}', {'v': 'Mike'}), col_names=None) + self.assertEqual(len(results), 0) + + def testStringSetIndexing(self): + """Tests indexing of items in string set columns.""" + + emails = ['spencer.kimball@emailscrubbed.com', 'spencer@goviewfinder.com', + 'petermattis@emailscrubbed.com', 'peter.mattis@gmail.com', 'peter@goviewfinder.com', + 'brian.mcginnis@emailscrubbed.com', 'brian@goviewfinder.com', + 'chris.schoenbohm@emailscrubbed.com', 'chris@goviewfinder.com'] + + # Create a bunch of contacts with one or two identities. + timestamp = util.GetCurrentTimestamp() + for email in emails: + for email2 in emails: + contact = Contact.CreateFromKeywords(1, + [('Email:' + email, None), ('Email:' + email2, None)], + timestamp, + Contact.GMAIL) + self._RunAsync(contact.Update, self._client) + + for email in emails: + q_contacts = self._RunAsync(Contact.IndexQuery, + self._client, + ('contact.identities={i}', {'i': 'Email:' + email}), + col_names=None) + logging.debug('querying for %s=%s yielded %d matches' % ('identities', 'Email:' + email, len(q_contacts))) + for contact in q_contacts: + self.assertTrue('Email:' + email in contact.identities) + self.assertEqual(len(q_contacts), len(emails) * 2 - 1) + + @async_test + def testRealTimeIndexing(self): + """Tests index updates in real-time.""" + def _QueryAndVerify(p, barrier_cb, query, is_in): + def _Verify(keys): + ids = [key.hash_key for key in keys] + if is_in: + self.assertTrue(p.photo_id in ids) + else: + self.assertFalse(p.photo_id in ids) + barrier_cb() + Photo.IndexQueryKeys(self._client, query, callback=_Verify) + + def _OnUpdate(p): + with util.Barrier(self.stop) as b: + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'Class'}), False) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'reunion'}), False) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': '1992'}), True) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'culumbia'}), True) + + def _Update(p): + p.caption = 'Columbia High School c.o. 1992' + p.Update(self._client, callback=partial(_OnUpdate, p)) + + photo_id = Photo.ConstructPhotoId(time.time(), 1, 1) + p = self.UpdateDBObject(Photo, user_id=self._user.user_id, + photo_id=photo_id, caption='Class of 1992 reunion') + + with util.Barrier(partial(_Update, p)) as b: + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'reunion'}), True) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': '1992'}), True) + + @async_test + @unittest.skipIf(platform.python_implementation() == 'PyPy', 'metaphone queries broken on pypy') + def testMetaphoneQueries(self): + """Tests metaphone queries.""" + def _QueryAndVerify(p, barrier_cb, query_expr, match): + def _Verify(keys): + ids = [key.hash_key for key in keys] + if match: + self.assertTrue(p.photo_id in ids) + else: + self.assertFalse(ids) + barrier_cb() + Photo.IndexQueryKeys(self._client, query_expr, callback=_Verify) + + photo_id = Photo.ConstructPhotoId(time.time(), 1, 1) + p = self.UpdateDBObject(Photo, user_id=self._user.user_id, + photo_id=photo_id, caption='Summer in East Hampton') + + with util.Barrier(self.stop) as b: + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'summer'}), True) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'sumer'}), True) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'summa'}), False) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'sum'}), False) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'hamton'}), False) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'hamptons'}), True) + _QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {'c': 'hammpton'}), True) + + # Disabled because we removed location secondary index from Episode table. + @async_test + def disabled_t_estLocationQueries(self): + """Tests location queries.""" + def _QueryAndVerify(episode_ids, barrier_cb, loc_search, matches): + def _Verify(keys): + ids = [key.hash_key for key in keys] + self.assertEqual(len(ids), len(matches)) + [self.assertTrue(episode_ids[m] in ids) for m in matches] + barrier_cb() + + Episode.IndexQueryKeys(self._client, 'episode.location="%f,%f,%f"' % \ + (loc_search[0], loc_search[1], loc_search[2]), callback=_Verify) + + def _OnCreate(locations, episodes): + with util.Barrier(self.stop) as b: + episode_ids = dict([(v.title, v.episode_id) for v in episodes]) + # Exact search. + _QueryAndVerify(episode_ids, b.Callback(), + Location(40.727657, -73.994583, 30), ['kimball ph']) + _QueryAndVerify(episode_ids, b.Callback(), + Location(41.044048, -71.950622, 100), ['surf lodge']) + # A super-small search area, centered in middle of Great Jones Alley. + _QueryAndVerify(episode_ids, b.Callback(), + Location(40.727267, -73.994443, 10), []) + # Widen the search area to 50m, centered in middle of Great Jones Alley. + _QueryAndVerify(episode_ids, b.Callback(), + Location(40.727267, -73.994443, 50), ['kimball ph', 'bond st sushi']) + # Union square with a 2km radius. + _QueryAndVerify(episode_ids, b.Callback(), + Location(40.736462, -73.990517, 2000), + ['kimball ph', 'bond st sushi', 'viewfinder', 'soho house', 'google']) + # The Dominican Republic. + _QueryAndVerify(episode_ids, b.Callback(), + Location(19.041349, -70.427856, 75000), ['casa kimball']) + # The Caribbean. + _QueryAndVerify(episode_ids, b.Callback(), + Location(22.593726, -76.662598, 800000), ['casa kimball', 'atlantis']) + # Long Island. + _QueryAndVerify(episode_ids, b.Callback(), Location(40.989228, -72.144470, 40000), + ['kimball east', 'surf lodge']) + + locations = {'kimball ph': Location(40.727657, -73.994583, 50.0), + 'bond st sushi': Location(40.726901, -73.994358, 50.0), + 'viewfinder': Location(40.720169, -73.998756, 200.0), + 'soho house': Location(40.740616, -74.005880, 200.0), + 'google': Location(40.740974, -74.002115, 500.0), + 'kimball east': Location(41.034184, -72.210603, 50.0), + 'surf lodge': Location(41.044048, -71.950622, 100.0), + 'casa kimball': Location(19.636848, -69.896602, 100.0), + 'atlantis': Location(25.086104, -77.323065, 1000.0)} + with util.ArrayBarrier(partial(_OnCreate, locations)) as b: + device_episode_id = 0 + for place, location in locations.items(): + device_episode_id += 1 + timestamp = time.time() + episode_id = Episode.ConstructEpisodeId(timestamp, 1, device_episode_id) + episode = Episode.CreateFromKeywords(timestamp=timestamp, + episode_id=episode_id, user_id=self._user.user_id, + viewpoint_id=self._user.private_vp_id, + publish_timestamp=timestamp, + title=place, location=location) + episode.Update(self._client, b.Callback()) + + # Disabled because we removed placemark secondary index from Episode table. + @async_test + def disabled_t_estPlacemarkQueries(self): + """Tests placemark queries.""" + def _QueryAndVerify(episode_ids, barrier_cb, search, matches): + def _Verify(keys): + ids = [key.hash_key for key in keys] + self.assertEqual(len(ids), len(matches)) + [self.assertTrue(episode_ids[m] in ids) for m in matches] + barrier_cb() + + Episode.IndexQueryKeys(self._client, ('episode.placemark={s}', {'s': search}), callback=_Verify) + + def _OnCreate(locations, episodes): + with util.Barrier(self.stop) as b: + episode_ids = dict([(v.title, v.episode_id) for v in episodes]) + _QueryAndVerify(episode_ids, b.Callback(), 'Broadway', ['kimball ph']) + _QueryAndVerify(episode_ids, b.Callback(), '682 Broadway', ['kimball ph']) + _QueryAndVerify(episode_ids, b.Callback(), 'Broadway 682', []) + _QueryAndVerify(episode_ids, b.Callback(), 'new york, ny, united states', + ['kimball ph', 'bond st sushi', 'viewfinder', 'soho house', 'google']) + _QueryAndVerify(episode_ids, b.Callback(), 'new york, ny', + ['kimball ph', 'bond st sushi', 'viewfinder', 'soho house', 'google']) + _QueryAndVerify(episode_ids, b.Callback(), 'NY, United States', + ['kimball ph', 'bond st sushi', 'viewfinder', 'soho house', 'google', + 'kimball east', 'surf lodge']) + _QueryAndVerify(episode_ids, b.Callback(), 'United States', + ['kimball ph', 'bond st sushi', 'viewfinder', 'soho house', 'google', + 'kimball east', 'surf lodge']) + _QueryAndVerify(episode_ids, b.Callback(), 'Bahamas', ['atlantis']) + _QueryAndVerify(episode_ids, b.Callback(), 'Dominican', ['casa kimball']) + _QueryAndVerify(episode_ids, b.Callback(), 'Dominican Republic', ['casa kimball']) + _QueryAndVerify(episode_ids, b.Callback(), 'Cabrera', ['casa kimball']) + _QueryAndVerify(episode_ids, b.Callback(), 'DR', ['casa kimball']) + + locations = {'kimball ph': Placemark('US', 'United States', 'NY', 'New York', + 'NoHo', 'Broadway', '682'), + 'bond st sushi': Placemark('US', 'United States', 'NY', 'New York', + 'NoHo', 'Bond St', '6'), + 'viewfinder': Placemark('US', 'United States', 'NY', 'New York', + 'SoHo', 'Grand St', '154'), + 'soho house': Placemark('US', 'United States', 'NY', 'New York', + 'Meatpacking District', '9th Avenue', '29-35'), + 'google': Placemark('US', 'United States', 'NY', 'New York', + 'Chelsea', '8th Avenue', '111'), + 'kimball east': Placemark('US', 'United States', 'NY', 'East Hampton', + 'Northwest Harbor', 'Milina', '35'), + 'surf lodge': Placemark('US', 'United States', 'NY', 'Montauk', + '', 'Edgemere St', '183'), + 'casa kimball': Placemark('DR', 'Dominican Republic', 'Maria Trinidad Sanchez', + 'Cabrera', 'Orchid Bay Estates', '', '5-6'), + 'atlantis': Placemark('BS', 'Bahamas', '', 'Paradise Island', '', '', '')} + with util.ArrayBarrier(partial(_OnCreate, locations)) as b: + device_episode_id = 0 + for place, placemark in locations.items(): + device_episode_id += 1 + timestamp = time.time() + episode_id = Episode.ConstructEpisodeId(timestamp, 1, device_episode_id) + episode = Episode.CreateFromKeywords(timestamp=timestamp, + episode_id=episode_id, user_id=self._user.user_id, + viewpoint_id=self._user.private_vp_id, + publish_timestamp=timestamp, + title=place, placemark=placemark) + episode.Update(self._client, b.Callback()) + + @async_test + def testQuerying(self): + """Tests querying of User objects.""" + def _QueryAndVerify(barrier_cb, query_expr, id_set): + def _Verify(keys): + ids = [key.hash_key for key in keys] + if not id_set: + self.assertFalse(ids) + else: + [self.assertTrue(i in id_set) for i in ids] + barrier_cb() + User.IndexQueryKeys(self._client, query_expr, + callback=_Verify) + + # Add given & family names to users created by base class. + spencer = self.UpdateDBObject(User, user_id=self._user.user_id, given_name='Spencer', family_name='Kimball') + andrew = self.UpdateDBObject(User, user_id=self._user2.user_id, given_name='Peter', family_name='Mattis') + + s_id = set([spencer.user_id]) + a_id = set([andrew.user_id]) + both_ids = s_id.union(a_id) + no_ids = set([]) + with util.Barrier(self.stop) as b: + _QueryAndVerify(b.Callback(), ('user.given_name={sp}', {'sp': 'spencer'}), s_id) + _QueryAndVerify(b.Callback(), ('user.given_name={sp}', {'sp': '\'spencer\''}), s_id) + _QueryAndVerify(b.Callback(), ('user.given_name={sp}', {'sp': '"spencer"'}), s_id) + _QueryAndVerify(b.Callback(), ('(user.given_name={sp})', {'sp': 'spencer'}), s_id) + _QueryAndVerify(b.Callback(), ('user.family_name={k}', {'k': 'kimball'}), both_ids) + _QueryAndVerify(b.Callback(), ('user.given_name={sp} & user.given_name={pe}', + {'sp': 'spencer', 'pe': 'peter'}), no_ids) + _QueryAndVerify(b.Callback(), ('(user.given_name={sp} & user.given_name={pe})', + {'sp': 'spencer', 'pe': 'peter'}), no_ids) + _QueryAndVerify(b.Callback(), ('user.given_name={sp} - user.given_name={pe}', + {'sp': 'spencer', 'pe': 'peter'}), s_id) + _QueryAndVerify(b.Callback(), ('user.given_name={sp} | user.given_name={pe}', + {'sp': 'spencer', 'pe': 'peter'}), both_ids) + _QueryAndVerify(b.Callback(), ('user.given_name={sp} - user.family_name={k}', + {'sp': 'spencer', 'k': 'kimball'}), no_ids) + _QueryAndVerify(b.Callback(), ('user.email={sp}', {'sp': 'spencer'}), s_id) + _QueryAndVerify(b.Callback(), ('user.email={sp} & user.email={gm}', {'sp': 'spencer', 'gm': 'gmail'}), s_id) + _QueryAndVerify(b.Callback(), ('user.email={sp} & user.email={gm} & user.email=com', + {'sp': 'spencer', 'gm': 'gmail', 'c': 'com'}), s_id) + _QueryAndVerify(b.Callback(), ('user.email={gm} & user.email=com - user.email=spencer', + {'gm': 'gmail', 'c': 'com', 'sp': 'spencer'}), no_ids) + _QueryAndVerify(b.Callback(), ('user.email={c}', {'c': 'com'}), both_ids) + _QueryAndVerify(b.Callback(), ('user.email={em}', {'em': '"spencer.kimball@emailscrubbed.com"'}), s_id) + _QueryAndVerify(b.Callback(), ('user.given_name={sp} | user.given_name={pe} - user.email={gm}', + {'sp': 'spencer', 'pe': 'peter', 'gm': 'gmail'}), both_ids) + _QueryAndVerify(b.Callback(), ('(user.given_name={sp} | user.given_name={pe}) - user.email={gm}', + {'sp': 'spencer', 'pe': 'peter', 'gm': 'gmail'}), a_id) + + @async_test + def testRangeSupport(self): + """Tests start_key, end_key, and limit support in IndexQueryKeys + and IndexQuery. + """ + name = 'Rumpelstiltskin' + vp_id = 'v0' + + def _QueryAndVerify(cls, barrier_cb, query_expr, start_key, end_key, limit): + def _FindIndex(list, db_key): + for i, item in enumerate(list): + if item.GetKey() == db_key: + return i + return -1 + + def _Verify(results): + all_items, some_items, some_item_keys = results + + # Ensure that IndexQuery and IndexQueryKeys return consistent results. + assert len(some_items) == len(some_item_keys) + assert [u.GetKey() for u in some_items] == some_item_keys + + # Ensure that right subset was returned. + start_index = _FindIndex(all_items, start_key) + 1 if start_key is not None else 0 + end_index = _FindIndex(all_items, end_key) if end_key is not None else len(all_items) + if limit is not None and start_index + limit < end_index: + end_index = start_index + limit + + assert len(some_items) == end_index - start_index, (len(some_items), start_index, end_index) + for expected_item, actual_item in zip(all_items[start_index:end_index], some_items): + expected_dict = expected_item._asdict() + actual_dict = actual_item._asdict() + self.assertEqual(expected_dict, actual_dict) + + barrier_cb() + + with util.ArrayBarrier(_Verify) as b: + cls.IndexQuery(self._client, query_expr, None, b.Callback(), limit=None) + cls.IndexQuery(self._client, query_expr, None, b.Callback(), + start_index_key=start_key, end_index_key=end_key, limit=limit) + cls.IndexQueryKeys(self._client, query_expr, b.Callback(), + start_index_key=start_key, end_index_key=end_key, limit=limit) + + def _RunQueries(cls, query_expr, hash_key_25, hash_key_75, callback): + with util.Barrier(callback) as b: + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=None, end_key=None, limit=None) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=None, end_key=None, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=None, end_key=hash_key_75, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=None, end_key=hash_key_25, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=hash_key_25, end_key=None, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=hash_key_75, end_key=None, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=hash_key_25, end_key=hash_key_75, limit=50) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=hash_key_25, end_key=hash_key_75, limit=1) + _QueryAndVerify(cls, b.Callback(), query_expr, start_key=hash_key_25, end_key=hash_key_75, limit=100) + + # Create 90 users all with the same given name, and 90 followers for the same viewpoint, + # and 90 followers with same adding_user_id. + for i in xrange(90): + user_id = i + 10 + self.UpdateDBObject(User, given_name=name, user_id=user_id, signing_key={}) + self.UpdateDBObject(Follower, user_id=user_id, viewpoint_id=vp_id) + + with util.Barrier(self.stop) as b: + _RunQueries(User, ('user.given_name={n}', {'n': name}), DBKey(25, None), DBKey(75, None), b.Callback()) + _RunQueries(Follower, ('follower.viewpoint_id={id}', {'id': vp_id}), DBKey(25, vp_id), DBKey(75, vp_id), + b.Callback()) + + def testUnicode(self): + """Test various interesting Unicode characters.""" + base_name = escape.utf8(u'ààà朋å‹ä½ å¥½abc123\U00010000\U00010000\x00\x01\b\n\t ') + timestamp = time.time() + contact_id_lookup = dict() + + def _CreateContact(index): + name = base_name + str(index) + identity_key = 'Email:%s' % name + return Contact.CreateFromKeywords(100, [(identity_key, None)], timestamp, Contact.GMAIL, name=name) + + def _VerifyContacts(query_expr, start_key, end_key, exp_indexes): + actual_contacts = self._RunAsync(Contact.IndexQuery, self._client, query_expr, None, + start_index_key=start_key, end_index_key=end_key) + self.assertEqual(len(exp_indexes), len(actual_contacts)) + for expected, actual in zip([_CreateContact(i) for i in exp_indexes], actual_contacts): + self.assertEqual(expected._asdict(), actual._asdict()) + + # Create 3 contacts under user 100 in the db. + for i in xrange(3): + contact = _CreateContact(i) + contact_id_lookup[i] = contact.contact_id + self._RunAsync(contact.Update, self._client) + + # Get contact by identity. + identity_key = 'Email:%s' % base_name + _VerifyContacts(('contact.identities={i}', {'i': identity_key + '0'}), None, None, [0]) + + # Get multiple contacts. + _VerifyContacts(('contact.identities={i} | contact.identities={i2}', + {'i': identity_key + '0', 'i2': identity_key + '1'}), + None, None, [1, 0]) + + # Get contact with start key. + sort_key = Contact.CreateSortKey(contact_id_lookup[1], timestamp) + _VerifyContacts(('contact.identities={i} | contact.identities={i2}', + {'i': identity_key + '0', 'i2': identity_key + '1'}), + DBKey(100, sort_key), None, [0]) + + # Get contact with end key. + sort_key = Contact.CreateSortKey(contact_id_lookup[0], timestamp) + _VerifyContacts(('contact.identities={i} | contact.identities={i2}', + {'i': identity_key + '0', 'i2': identity_key + '1'}), + None, DBKey(100, sort_key), [1]) diff --git a/backend/db/test/job_test.py b/backend/db/test/job_test.py new file mode 100644 index 0000000..ac59c64 --- /dev/null +++ b/backend/db/test/job_test.py @@ -0,0 +1,135 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for Job class. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import time + +from viewfinder.backend.base import constants +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db.job import Job +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType + +from base_test import DBBaseTestCase + +class JobTestCase(DBBaseTestCase): + def testLocking(self): + """Test basic locking mechanism.""" + job1 = Job(self._client, 'test_job') + self.assertTrue(self._RunAsync(job1.AcquireLock)) + + job2 = Job(self._client, 'test_job') + self.assertFalse(self._RunAsync(job2.AcquireLock)) + + # Abandon job1 lock. We never do this on real jobs, so manually clear the lock. + self._RunAsync(job1._lock.Abandon, self._client) + job1._lock = None + + # Set detect_abandonment=False: failure. + self.assertFalse(self._RunAsync(job2.AcquireLock, detect_abandonment=False)) + self.assertFalse(self._RunAsync(job2.AcquireLock, detect_abandonment=False)) + + # Now allow abandoned lock acquisition. + self.assertTrue(self._RunAsync(job2.AcquireLock)) + self.assertFalse(self._RunAsync(job1.AcquireLock)) + self._RunAsync(job2.ReleaseLock) + + # Job1 grabs the lock again. + self.assertTrue(self._RunAsync(job1.AcquireLock)) + self._RunAsync(job1.ReleaseLock) + + def testMetrics(self): + """Test fetching/writing metrics.""" + # Job being tested. + job = Job(self._client, 'test_job') + prev_runs = self._RunAsync(job.FindPreviousRuns) + self.assertEqual(len(prev_runs), 0) + + # Unrelated job with a different name. Run entries should not show up under 'test_job'. + other_job = Job(self._client, 'other_test_job') + other_job.Start() + self._RunAsync(other_job.RegisterRun, Job.STATUS_SUCCESS) + other_job.Start() + self._RunAsync(other_job.RegisterRun, Job.STATUS_FAILURE) + + # Calling RegisterRun without first calling Start fails because the start_time is not set. + self.assertIsNone(job._start_time) + self.assertRaises(AssertionError, self._RunAsync, job.RegisterRun, Job.STATUS_SUCCESS) + + job.Start() + self.assertIsNotNone(job._start_time) + # Overwrite it for easier testing. + start_time = job._start_time = int(time.time() - (constants.SECONDS_PER_WEEK + constants.SECONDS_PER_HOUR)) + + # Write run summary with extra stats. + stats = DotDict() + stats['foo.bar'] = 5 + stats['baz'] = 'test' + self._RunAsync(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats, failure_msg='foo') + # start_time is reset to prevent multiple calls to RegisterRun. + self.assertIsNone(job._start_time) + self.assertRaises(AssertionError, self._RunAsync, job.RegisterRun, Job.STATUS_SUCCESS) + + end_time = int(time.time()) + # Default search is "runs started in the past week". + prev_runs = self._RunAsync(job.FindPreviousRuns) + self.assertEqual(len(prev_runs), 0) + # Default search is for successful runs. + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10)) + self.assertEqual(len(prev_runs), 1) + self.assertEqual(prev_runs[0]['start_time'], start_time) + self.assertAlmostEqual(prev_runs[0]['end_time'], end_time, delta=10) + self.assertEqual(prev_runs[0]['status'], Job.STATUS_SUCCESS) + self.assertEqual(prev_runs[0]['stats.foo.bar'], 5) + self.assertEqual(prev_runs[0]['stats.baz'], 'test') + # failure_msg does nothing when status is SUCCESS. + self.assertTrue('failure_msg' not in prev_runs[0]) + + # Search for failed runs. + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10), status=Job.STATUS_FAILURE) + self.assertEqual(len(prev_runs), 0) + + # Create a failed job summary. + job.Start() + start_time2 = job._start_time = int(time.time() - constants.SECONDS_PER_HOUR) + self._RunAsync(job.RegisterRun, Job.STATUS_FAILURE, failure_msg='stack trace') + + # Find previous runs using a variety of filters. + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10), status=Job.STATUS_SUCCESS) + self.assertEqual(len(prev_runs), 1) + self.assertEqual(prev_runs[0]['start_time'], start_time) + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10), status=Job.STATUS_FAILURE) + self.assertEqual(len(prev_runs), 1) + self.assertEqual(prev_runs[0]['status'], Job.STATUS_FAILURE) + self.assertEqual(prev_runs[0]['failure_msg'], 'stack trace') + self.assertEqual(prev_runs[0]['start_time'], start_time2) + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10)) + self.assertEqual(len(prev_runs), 2) + self.assertEqual(prev_runs[0]['start_time'], start_time) + self.assertEqual(prev_runs[1]['start_time'], start_time2) + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time2 - 10)) + self.assertEqual(len(prev_runs), 1) + self.assertEqual(prev_runs[0]['start_time'], start_time2) + prev_runs = self._RunAsync(job.FindPreviousRuns, start_timestamp=(start_time - 10), limit=1) + self.assertEqual(len(prev_runs), 1) + self.assertEqual(prev_runs[0]['start_time'], start_time2) + + # Find last successful run with optional payload key/value. + prev_success = self._RunAsync(job.FindLastSuccess, start_timestamp=(start_time - 10)) + self.assertIsNotNone(prev_success) + self.assertEqual(prev_success['stats.foo.bar'], 5) + prev_success = self._RunAsync(job.FindLastSuccess, start_timestamp=(start_time - 10), with_payload_key='stats.baz') + self.assertIsNotNone(prev_success) + self.assertEqual(prev_success['stats.foo.bar'], 5) + prev_success = self._RunAsync(job.FindLastSuccess, start_timestamp=(start_time - 10), with_payload_key='stats.bar') + self.assertIsNone(prev_success) + prev_success = self._RunAsync(job.FindLastSuccess, start_timestamp=(start_time - 10), + with_payload_key='stats.baz', with_payload_value='test') + self.assertIsNotNone(prev_success) + self.assertEqual(prev_success['stats.foo.bar'], 5) + prev_success = self._RunAsync(job.FindLastSuccess, start_timestamp=(start_time - 10), + with_payload_key='stats.baz', with_payload_value='test2') + self.assertIsNone(prev_success) diff --git a/backend/db/test/local_client_test.py b/backend/db/test/local_client_test.py new file mode 100644 index 0000000..11bd56c --- /dev/null +++ b/backend/db/test/local_client_test.py @@ -0,0 +1,443 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for local emulation of DynamoDB client. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import random + +from tornado import options +from viewfinder.backend.base.testing import async_test, BaseTestCase +from viewfinder.backend.db.db_client import DBKey, DBKeySchema, UpdateAttr, BatchGetRequest, RangeOperator, ScanFilter +from viewfinder.backend.db.local_client import LocalClient +from viewfinder.backend.db.schema import Schema, Table, Column, HashKeyColumn, RangeKeyColumn + +_hash_key_schema = DBKeySchema(name='test_hk', value_type='N') +_range_key_schema = DBKeySchema(name='test_rk', value_type='N') + +test_SCHEMA = Schema([ + Table('LocalTest', 'lt', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + RangeKeyColumn('test_rk', 'test_rk', 'N'), + Column('num', 'num', 'N'), + Column('str', 'str', 'S'), + Column('num_set', 'num_set', 'NS'), + Column('str_set', 'str_set', 'SS')]), + + Table('LocalTest2', 'lt2', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + Column('num', 'num', 'N'), + Column('str', 'str', 'S'), + Column('num_set', 'num_set', 'NS'), + Column('str_set', 'str_set', 'SS')]), + + Table('Cond', 'co', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + Column('attr1', 'attr1', 'S')]), + + Table('Cond2', 'co2', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + RangeKeyColumn('test_rk', 'test_rk', 'N'), + Column('attr1', 'attr1', 'S')]), + + Table('RangeTest', 'rt', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + RangeKeyColumn('test_rk', 'test_rk', 'N'), + Column('attr1', 'attr1', 'N'), + Column('attr2', 'attr2', 'S')]), + + Table('Errors', 'err', read_units=10, write_units=5, + columns=[HashKeyColumn('test_hk', 'test_hk', 'N'), + RangeKeyColumn('test_rk', 'test_rk', 'N'), + Column('attr1', 'attr1', 'S')]), + ]) + + +class LocalClientTestCase(BaseTestCase): + def setUp(self): + """Sets up _client as a test emulation of DynamoDB. Creates the full + database schema, a test user, and two devices (one for mobile, one + for web-application). + """ + super(LocalClientTestCase, self).setUp() + options.options.localdb_dir = '' + self._client = LocalClient(test_SCHEMA) + test_SCHEMA.VerifyOrCreate(self._client, self.stop) + self.wait() + + @async_test + def testTables(self): + # Empty list tables. + lr = self._client.ListTables(callback=None) + self.assertTrue('NewTable1' not in lr.tables) + # Create a table. + cr = self._client.CreateTable('NewTable1', _hash_key_schema, + _range_key_schema, 5, 10, callback=None) + self.assertEqual(cr.schema.status, 'CREATING') + # Create a second table. + cr = self._client.CreateTable('NewTable2', _hash_key_schema, + None, 5, 10, callback=None) + self.assertEqual(cr.schema.status, 'CREATING') + # List tables. + lr = self._client.ListTables(callback=None) + self.assertTrue('NewTable1' in lr.tables) + self.assertTrue('NewTable2' in lr.tables) + # Describe tables. + for table in lr.tables: + dr = self._client.DescribeTable(table, callback=None) + self.assertEqual(dr.schema.status, 'ACTIVE') + # Delete tables. + cr1 = self._client.DeleteTable(table='NewTable1', callback=None) + cr2 = self._client.DeleteTable(table='NewTable2', callback=None) + self.assertEqual(cr1.schema.status, 'DELETING') + self.assertEqual(cr2.schema.status, 'DELETING') + # Verify tables are gone. + lr = self._client.ListTables(callback=None) + self.assertTrue('NewTable1' not in lr.tables) + self.assertTrue('NewTable2' not in lr.tables) + self.stop() + + @async_test + def testErrors(self): + self.assertRaises(Exception, self._client.DeleteTable, + table='NonExistent', callback=None) + # Missing range key. + self.assertRaises(Exception, self._client.PutItem, table='Errors', + key=DBKey(hash_key=1, range_key=None), attributes={'attr1': 1}) + + # Bad hash key type. + self.assertRaises(Exception, self._client.PutItem, table='Errors', + key=DBKey(hash_key='a', range_key=1), attributes={'attr1': 1}) + + # Bad range key. + self.assertRaises(Exception, self._client.PutItem, table='Errors', + key=DBKey(hash_key=1, range_key='a'), attributes={'attr1': 1}) + + # Success + self._client.PutItem(table='Errors', key=DBKey(hash_key=1, range_key=1), + attributes={'attr1': 1}, callback=None) + + # Missing hash key on get. + self.assertRaises(Exception, self._client.GetItem, table='Errors', + key=DBKey(hash_key=None, range_key=1), attributes=['attr1']) + + # Missing range key on get. + self.assertRaises(Exception, self._client.GetItem, table='Errors', + key=DBKey(hash_key=1, range_key=None), attributes=['attr1']) + + self.stop() + + @async_test + def testPutAndGet(self): + # Add some new items with attributes. + items = {} + for i in xrange(100): + attrs = {'num': i * 2, 'str': 'test-%d' % i, 'num_set': set([i, (i + 1) * 2, (i + i) ** 2]), + 'str_set': set(['s%d' % i, 's%d' % (i + 1) * 2, 's%d' % (i + 1) ** 2])} + items[i] = attrs + result = self._client.PutItem(table='LocalTest2', + key=DBKey(hash_key=i, range_key=None), attributes=attrs, + return_values='ALL_OLD', callback=None) + self.assertEqual(result.write_units, 1) + self.assertFalse(result.return_values) + + # Get the items. + for i in xrange(100): + fetch_list = items[i].keys() + random.shuffle(items[i].keys()) + fetch_list = fetch_list[0:random.randint(1, len(fetch_list))] + result = self._client.GetItem(table='LocalTest2', key=DBKey(hash_key=i, range_key=None), + attributes=fetch_list, callback=None) + self.assertEqual(result.read_units, 1) + self.assertEqual(len(result.attributes), len(fetch_list)) + for attr in fetch_list: + self.assertEqual(result.attributes[attr], items[i][attr]) + + # Batch get the items + batch_dict = {'LocalTest2': BatchGetRequest(keys=[DBKey(i, None) for i in xrange(100)], + attributes=['test_hk', 'num', 'str', 'num_set', 'str_set'], + consistent_read=True)} + result = self._client.BatchGetItem(batch_dict, callback=None)['LocalTest2'] + self.assertEqual(result.read_units, 100) + self.assertEqual(len(result.items), 100) + for i in xrange(100): + self.assertEqual(result.items[i], items[i]) + + # Delete the items. + for i in xrange(100): + result = self._client.DeleteItem(table='LocalTest2', key=DBKey(hash_key=i, range_key=None), + callback=None) + for i in xrange(100): + self.assertRaises(Exception, self._client.GetItem, table='LocalTest2', + key=DBKey(hash_key=i, range_key=None), attributes=None, callback=None) + + self.stop() + + @async_test + def testConditionalPutAndUpdate(self): + # Try to put an item with an attribute which must exist. + self.assertRaises(Exception, self._client.PutItem, table='Cond', + key=DBKey(hash_key=1, range_key=None), expected={'attr1': 'val'}, + attributes={'attr1': 'new_val'}) + # Now add this item and try again. + self._client.PutItem(table='Cond', key=DBKey(hash_key=1, range_key=None), + attributes={'attr1': 'new_val'}, callback=None) + # But with wrong value. + self.assertRaises(Exception, self._client.PutItem, table='Cond', + key=DBKey(hash_key=1, range_key=None), expected={'attr1': 'val'}, + attributes={'attr1': 'new_val'}) + # Now with correct value. + self._client.PutItem(table='Cond', key=DBKey(hash_key=1, range_key=None), + expected={'attr1': 'new_val'}, attributes={'attr1': 'even_newer_val'}, + callback=None) + + # Try to put an item which already exists, but which mustn't. + self.assertRaises(Exception, self._client.PutItem, table='Cond', + key=DBKey(hash_key=1, range_key=None), + expected={_hash_key_schema.name: False}, + attributes={'attr1': 'new_val'}, callback=None) + + # Try with a composite key object. + self.assertRaises(Exception, self._client.PutItem, table='Cond2', + key=DBKey(hash_key=1, range_key=1), expected={'attr1': 'val'}, + attributes={'attr1': 'new_val'}) + self._client.PutItem(table='Cond2', key=DBKey(hash_key=1, range_key=1), + attributes={'attr1': 'new_val'}, callback=None) + self.assertRaises(Exception, self._client.PutItem, table='Cond2', + key=DBKey(hash_key=1, range_key=1), + expected={_hash_key_schema.name: False}, + attributes={'attr1': 'even_newer_val'}, callback=None) + + self.stop() + + @async_test + def testUpdate(self): + def _VerifyUpdate(updates, get_attrs, new_attrs, rv, rvs): + update_res = self._client.UpdateItem( + table='LocalTest2', key=DBKey(hash_key=1, range_key=None), + callback=None, attributes=updates, return_values=rv) + for k, v in rvs.items(): + self.assertEqual(update_res.return_values[k], v) + + get_res = self._client.GetItem( + table='LocalTest2', key=DBKey(hash_key=1, range_key=None), + callback=None, attributes=get_attrs) + + if get_res.attributes: + self.assertEqual(len(get_res.attributes), len(new_attrs)) + for k, v in new_attrs.items(): + self.assertEqual(get_res.attributes[k], v) + else: + self.assertEqual(get_res.attributes, {}) + + # Add a test item. + attrs = {'num': 1, 'str': 'test', 'num_set': list([1, 2, 3]), + 'str_set': list(['a', 'b', 'c'])} + self._client.PutItem(table='LocalTest2', key=DBKey(hash_key=1, range_key=None), + attributes=attrs, callback=None) + + # Update values, getting different return values type on each iteration. + _VerifyUpdate({'num': UpdateAttr(value=2, action='PUT')}, + ['num'], {'num': 2}, 'NONE', {}) + + _VerifyUpdate({'num': UpdateAttr(value=2, action='ADD')}, + ['num'], {'num': 4}, 'UPDATED_NEW', {'num': 4}) + + _VerifyUpdate({'num': UpdateAttr(value=2, action='DELETE')}, + ['num'], None, 'UPDATED_OLD', {'num': 4}) + + _VerifyUpdate({'num': UpdateAttr(value= -1, action='ADD'), + 'num_set': UpdateAttr(value=list([4, 5, 6]), action='PUT')}, + ['num', 'num_set'], {'num':-1, 'num_set': list([4, 5, 6])}, + 'ALL_OLD', {'str': 'test', 'num_set': list([1, 2, 3]), + 'str_set': list(['a', 'b', 'c'])}) + + _VerifyUpdate({'str': UpdateAttr(value='new_test', action='PUT'), + 'num_set': UpdateAttr(value=list([2, 3, 4]), action='ADD')}, + ['str', 'num_set'], {'str': 'new_test', 'num_set': list([2, 3, 4, 5, 6])}, + 'ALL_NEW', {'num':-1, 'str': 'new_test', 'num_set': list([2, 3, 4, 5, 6]), + 'str_set': list(['a', 'b', 'c'])}) + + _VerifyUpdate({'str_set': UpdateAttr(value=list(['a', 'd']), action='DELETE'), + 'num_set': UpdateAttr(value=list([3, 4, 6, 100]), action='DELETE'), + 'unknown': UpdateAttr(value=list([100]), action='DELETE')}, + ['str_set', 'num_set'], + {'str_set': list(['b', 'c']), 'num_set': list([2, 5])}, 'NONE', {}) + + _VerifyUpdate({'str_set': UpdateAttr(value=None, action='DELETE'), + 'str': UpdateAttr(value=None, action='DELETE')}, + ['str_set', 'str'], {}, 'NONE', {}) + + self.stop() + + @async_test + def testRangeQuery(self): + items = {} + hash_key = 1 + for i in xrange(100): + items[i] = {'attr1': i * 2, 'attr2': 'test-%d' % i} + result = self._client.PutItem('RangeTest', key=DBKey(hash_key=hash_key, range_key=i), + attributes=items[i], callback=None) + self.assertEqual(result.write_units, 1) + self.assertFalse(result.return_values) + + def _VerifyRange(r_op, limit, forward, start_key, exp_keys, exp_last_key): + """Returns the result""" + result = self._client.Query(table='RangeTest', hash_key=hash_key, + range_operator=r_op, callback=None, + attributes=['test_hk', 'test_rk', 'attr1', 'attr2'], + limit=limit, scan_forward=forward, + excl_start_key=(DBKey(hash_key, start_key) if start_key else None)) + self.assertEqual(len(exp_keys), result.count) + self.assertEqual(len(exp_keys), len(result.items)) + self.assertEqual(exp_keys, [item['test_rk'] for item in result.items]) + for item in result.items: + self.assertEqual(items[item['test_rk']]['attr1'], item['attr1']) + self.assertEqual(items[item['test_rk']]['attr2'], item['attr2']) + if exp_last_key is not None: + self.assertEqual(DBKey(hash_key, exp_last_key), result.last_key) + else: + self.assertTrue(result.last_key is None) + + _VerifyRange(None, limit=None, forward=True, start_key=None, + exp_keys=range(0, 100), exp_last_key=None) + + _VerifyRange(None, limit=0, forward=True, start_key=None, + exp_keys=[], exp_last_key=None) + + _VerifyRange(RangeOperator(key=[25, 75], op='BETWEEN'), limit=None, + forward=True, start_key=None, exp_keys=range(25, 76), exp_last_key=None) + + _VerifyRange(RangeOperator(key=[25, 75], op='BETWEEN'), limit=25, + forward=True, start_key=None, exp_keys=range(25, 50), exp_last_key=49) + + _VerifyRange(RangeOperator(key=[25, 75], op='BETWEEN'), limit=None, + forward=False, start_key=None, exp_keys=range(75, 24, -1), exp_last_key=None) + + _VerifyRange(RangeOperator(key=[25, 75], op='BETWEEN'), limit=25, + forward=False, start_key=None, exp_keys=range(75, 50, -1), exp_last_key=51) + + _VerifyRange(RangeOperator(key=[25], op='GT'), limit=1, + forward=True, start_key=None, exp_keys=[26], exp_last_key=26) + + _VerifyRange(RangeOperator(key=[25], op='GT'), limit=1, + forward=True, start_key=26, + exp_keys=[27], exp_last_key=27) + + _VerifyRange(RangeOperator(key=[50], op='LT'), limit=10, + forward=False, start_key=48, + exp_keys=range(47, 37, -1), exp_last_key=38) + + _VerifyRange(RangeOperator(key=[10], op='GE'), limit=None, + forward=True, start_key=None, + exp_keys=range(10, 100), exp_last_key=None) + + _VerifyRange(None, limit=10, forward=True, start_key=None, + exp_keys=range(0, 10), exp_last_key=9) + + self.stop() + + @async_test + def testRangeScan(self): + items = {} + for h in xrange(2): + items[h] = {} + for r in xrange(2): + items[h][r] = {'attr1': (h + r * 2), 'attr2': 'test-%d' % (h + r)} + result = self._client.PutItem('RangeTest', key=DBKey(hash_key=h, range_key=r), + attributes=items[h][r], callback=None) + self.assertEqual(result.write_units, 1) + self.assertFalse(result.return_values) + + def _VerifyScan(limit, start_key, exp_keys, exp_last_key): + result = self._client.Scan(table='RangeTest', callback=None, + attributes=['test_hk', 'test_rk', 'attr1', 'attr2'], + limit=limit, excl_start_key=start_key) + for item in result.items: + self.assertTrue((item['test_hk'], item['test_rk']) in exp_keys) + if exp_last_key is not None: + self.assertEqual(exp_last_key, result.last_key) + else: + self.assertTrue(result.last_key is None) + + _VerifyScan(None, None, set([(0, 0), (0, 1), (1, 0), (1, 1)]), None) + + self.stop() + + @async_test + def testScanFilter(self): + items = {} + for h in xrange(2): + items[h] = {} + for r in xrange(2): + items[h][r] = {'attr1': (h + r * 2), 'attr2': 'test-%d' % (h + r)} + self._client.PutItem('RangeTest', key=DBKey(hash_key=h, range_key=r), + attributes=items[h][r], callback=None) + + def _VerifyScan(scan_filter, exp_keys): + result = self._client.Scan(table='RangeTest', callback=None, + attributes=['test_hk', 'test_rk', 'attr1', 'attr2'], + scan_filter=scan_filter) + for item in result.items: + key = (item['test_hk'], item['test_rk']) + self.assertTrue(key in exp_keys) + exp_keys.remove(key) + self.assertEqual(len(exp_keys), 0) + + _VerifyScan(None, set([(0, 0), (0, 1), (1, 0), (1, 1)])) + _VerifyScan({'attr1': ScanFilter([0], 'EQ')}, set([(0, 0)])) + _VerifyScan({'attr1': ScanFilter([0], 'GE')}, set([(0, 0), (0, 1), (1, 0), (1, 1)])) + _VerifyScan({'attr1': ScanFilter([0], 'GT')}, set([(0, 1), (1, 0), (1, 1)])) + # Try two conditions. + _VerifyScan({'attr1': ScanFilter([0], 'EQ'), + 'attr2': ScanFilter(['test-0'], 'EQ')}, set([(0, 0)])) + _VerifyScan({'attr1': ScanFilter([0], 'EQ'), + 'attr2': ScanFilter(['test-1'], 'EQ')}, set([])) + + self.stop() + + +class LocalReadOnlyClientTestCase(BaseTestCase): + def setUp(self): + """Creates a read-only local client. + Manually flips the _read_only variable to populate the table, then flips it back. + """ + super(LocalReadOnlyClientTestCase, self).setUp() + options.options.localdb_dir = '' + self._client = LocalClient(test_SCHEMA, read_only=True) + + self._client._read_only = False + self._RunAsync(test_SCHEMA.VerifyOrCreate, self._client) + self._RunAsync(self._client.PutItem, table='LocalTest', key=DBKey(hash_key=1, range_key=1), attributes={'num': 1}) + self._client._read_only = True + + + def testMethods(self): + # Read-only methods: + self._RunAsync(self._client.ListTables) + self._RunAsync(self._client.DescribeTable, 'LocalTest') + self._RunAsync(self._client.GetItem, table='LocalTest', + key=DBKey(hash_key=1, range_key=1), attributes=['num']) + batch_dict = {'LocalTest': BatchGetRequest(keys=[DBKey(1, 1)], attributes=['num'], consistent_read=True)} + self._RunAsync(self._client.BatchGetItem, batch_dict) + self._RunAsync(self._client.Query, table='LocalTest', hash_key=1, range_operator=None, attributes=None) + self._RunAsync(self._client.Scan, table='LocalTest', attributes=None) + + # Mutating methods: + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.CreateTable, 'LocalTest', _hash_key_schema, _range_key_schema, 5, 10) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.DeleteTable, table='LocalTest') + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.PutItem, table='LocalTest', key=DBKey(hash_key=1, range_key=2), + attributes={'num': 1}) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.DeleteItem, table='LocalTest', key=DBKey(hash_key=1, range_key=2)) + + self.assertRaisesRegexp(AssertionError, 'request on read-only database', self._RunAsync, + self._client.UpdateItem, table='LocalTest', key=DBKey(hash_key=1, range_key=2), + attributes={'num': 1}) diff --git a/backend/db/test/lock_test.py b/backend/db/test/lock_test.py new file mode 100644 index 0000000..e645d11 --- /dev/null +++ b/backend/db/test/lock_test.py @@ -0,0 +1,219 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Lock class. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import time + +from datetime import timedelta +from functools import partial +from base_test import DBBaseTestCase +from viewfinder.backend.base.exceptions import LockFailedError +from viewfinder.backend.db.lock import Lock + + +class LockTestCase(DBBaseTestCase): + def testSimpleAcquire(self): + """Test acquiring a lock.""" + self._TryAcquire('my', '1234') + self._TryAcquire('my', '!@#%!@#$', resource_data='some data') + self._TryAcquire('my', '12-34', resource_data='the quick brown fox jumped over the lazy dog', + detect_abandonment=True) + + def testLockAlreadyAcquired(self): + """Test attempt to acquire a lock that is already owned.""" + lock = self._TryAcquire('op', 'andy', release_lock=False) + self._TryAcquire('op', 'andy', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK) + self._TryAcquire('op', 'andy', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK) + self._Release(lock) + self.assertEqual(lock.acquire_failures, 2) + + def testAbandonedLock(self): + """Test acquiring an abandoned lock.""" + lock = self._TryAcquire('op', 'id', detect_abandonment=True, release_lock=False) + lock.Abandon(self._client, self.stop) + self.wait() + self._TryAcquire('op', 'id', expected_status=Lock.ACQUIRED_ABANDONED_LOCK) + + def testRenewLock(self): + """Test that renewal mechanism is preventing lock abandonment.""" + Lock.ABANDONMENT_SECS = .3 + Lock.LOCK_RENEWAL_SECS = .1 + lock = self._TryAcquire('op', 'id', detect_abandonment=True, release_lock=False) + self.io_loop.add_timeout(timedelta(seconds=.6), self.stop) + self.wait() + self._TryAcquire('op', 'id', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK) + self._Release(lock) + Lock.ABANDONMENT_SECS = 60 + Lock.LOCK_RENEWAL_SECS = 30 + + def testRaceToAcquire(self): + """Test cases where multiple agents are racing to acquire locks.""" + lock_to_release = [] + + def _OnAcquire(update_func, lock, status): + lock_to_release.append(lock) + update_func() + + def _Race(update_func): + if len(lock_to_release) == 0: + # Win race to acquire lock. + Lock.TryAcquire(self._client, 'op', 'id', partial(_OnAcquire, update_func)) + elif len(lock_to_release) == 1: + # Win race to update acquire_failures. + Lock.TryAcquire(self._client, 'op', 'id', partial(_OnAcquire, update_func)) + else: + update_func() + + self._TryAcquire('op', 'id', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK, + test_hook=_Race) + + self._Release(lock_to_release[0]) + + def testRaceToUpdateReleasedLock(self): + """Test case where failed lock acquirer tries to update lock after it's been released.""" + + def _Race(lock_to_release, update_func): + # Release the acquired lock after the current attempt has queried the row for the lock + # and has its own instance of the lock. + lock_to_release.Release(self._client, callback=update_func) + + # First, acquire the lock. + lock_to_release = self._TryAcquire('op', 'id', release_lock=False) + + # Now, try to acquire, expecting failure because we'll release it in the test hook. + self._TryAcquire('op', 'id', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK, + test_hook=partial(_Race, lock_to_release), release_lock=False) + + # Now, check for lock row. There shouldn't be one. + lock_id = Lock.ConstructLockId('op', 'id') + lock = self._RunAsync(Lock.Query, self._client, lock_id, None, must_exist=False) + self.assertIsNone(lock, 'Lock row should not exist') + + def testRaceToTakeOver(self): + """Test cases where multiple agents are racing to take over an + abandoned lock. + """ + lock_to_release = [] + + def _OnAcquire(update_func, lock, status): + lock_to_release.append(lock) + update_func() + + def _Race(update_func): + if len(lock_to_release) == 0: + # Win race to acquire the abandoned lock. + Lock.TryAcquire(self._client, 'op', 'id', partial(_OnAcquire, update_func)) + else: + update_func() + + lock = self._TryAcquire('op', 'id', detect_abandonment=True, release_lock=False) + lock.expiration = time.time() - 1 + lock.Update(self._client, self.stop) + self.wait() + self._TryAcquire('op', 'id', expected_status=Lock.FAILED_TO_ACQUIRE_LOCK, + test_hook=_Race) + + self._Release(lock_to_release[0]) + + def testAcquireLock(self): + """Test Lock.Acquire function.""" + hit_exception = False + + # This should succeed. + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', None) + try: + # This should fail and raise an error because the lock is already taken. + self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', None) + try: + self.fail("Shouldn't reach this point in try.") + finally: + self.fail("Shouldn't reach this point in finally.") + except Exception as e: + hit_exception = True + self.assertEqual(type(e), LockFailedError) + finally: + self._RunAsync(lock.Release, self._client) + + self.assertTrue(lock.IsReleased()) + self.assertTrue(hit_exception) + + def testAcquireLockWithError(self): + """Test Lock.Acquire failure in try/except that encounters an error.""" + error_was_raised = False + + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', None) + try: + raise Exception('raise an error') + except Exception as e: + self.assertEqual(e.message, 'raise an error') + error_was_raised = True + finally: + self._RunAsync(lock.Release, self._client) + + self.assertTrue(error_was_raised) + self.assertTrue(lock.IsReleased()) + self.assertTrue(lock.AmOwner()) + + # Now, ensure that the lock can be aquired. + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', None) + self.assertTrue(lock.AmOwner()) + self.assertTrue(not lock.IsReleased()) + self._RunAsync(lock.Release, self._client) + + def testReaquireLock(self): + """Test that Lock.Acquire will succeed with orphaned lock when owner_id matches that of lock.""" + + # Create lock, but don't release so that it's 'orphaned'. + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', 'owner89') + self.assertTrue(lock.AmOwner()) + + # Now, try to acquire the same lock using the same owner_id and observe that it succeeds. + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', 'owner89') + self.assertTrue(lock.AmOwner()) + self._RunAsync(lock.Release, self._client) + + def testReleaseOtherOwnedLock(self): + """Test releasing a lock that's owned by a different agent.""" + lock = self._RunAsync(Lock.Acquire, self._client, 'tst', 'id0', 'owner89') + + # Read same lock into a new object. + lock2 = self._RunAsync(Lock.Query, self._client, lock.lock_id, None) + # Set a different owner_id on it: + lock2.owner_id = 'new_owner_id' + self._RunAsync(lock2.Update, self._client) + + # Now try to release (should fail). + self.assertRaises(LockFailedError, self._RunAsync, lock.Release, self._client) + + # Now, read it to demonstrate that it hasn't been released. + lock3 = self._RunAsync(Lock.Query, self._client, lock.lock_id, None) + + def _TryAcquire(self, resource_type, resource_id, expected_status=Lock.ACQUIRED_LOCK, + resource_data=None, detect_abandonment=False, release_lock=True, test_hook=None): + Lock._TryAcquire(self._client, resource_type, resource_id, + lambda lock, status: self.stop((lock, status)), + resource_data=resource_data, detect_abandonment=detect_abandonment, test_hook=test_hook) + lock, status = self.wait() + + if release_lock and status != Lock.FAILED_TO_ACQUIRE_LOCK: + self._Release(lock) + + self.assertEqual(status, expected_status) + + if status != Lock.FAILED_TO_ACQUIRE_LOCK: + self.assertEqual(Lock.DeconstructLockId(lock.lock_id), (resource_type, resource_id)) + self.assertIsNotNone(lock.owner_id) + if detect_abandonment: + self.assertAlmostEqual(lock.expiration, time.time() + Lock.ABANDONMENT_SECS, + delta=Lock.ABANDONMENT_SECS / 4) + self.assertEqual(lock.resource_data, resource_data) + + return lock + + def _Release(self, lock): + lock.Release(self._client, callback=self.stop) + self.assertTrue(lock.IsReleased) + self.wait() diff --git a/backend/db/test/metric_test.py b/backend/db/test/metric_test.py new file mode 100644 index 0000000..c6325a6 --- /dev/null +++ b/backend/db/test/metric_test.py @@ -0,0 +1,146 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Metric upload process. +""" + +__author__ = 'matt@emailscrubbed.com (Matt Tracy)' + +import json +import time + +from base_test import DBBaseTestCase +from functools import partial +from tornado.ioloop import IOLoop +from viewfinder.backend.base import counters, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.metric import Metric, AggregatedMetric, MetricInterval + +class MetricsTestCase(DBBaseTestCase): + @async_test + def testMetricsUploadTimer(self): + """Test the periodic metric upload system.""" + def _OnQueryMetric(min_metrics, max_metrics, metrics): + self.assertTrue(len(metrics) >= min_metrics and len(metrics) <= max_metrics, + '%d not in [%d-%d]' % (len(metrics), min_metrics, max_metrics)) + for m in metrics: + self.assertTrue(m.timestamp % 3 == 0) + payload = DotDict(json.loads(metrics[0].payload)) + keys = counters.counters.flatten().keys() + for k in keys: + self.assertTrue(k in payload, 'Payload did not contain record for counter %s' % k) + self.stop() + + start_time = time.time() + cluster_name = 'test_group_key' + interval = MetricInterval('test_interval', 3) + group_key = Metric.EncodeGroupKey(cluster_name, interval) + rate = counters.define_rate('metricstest.rate', 'Test rate') + + Metric.StartMetricUpload(self._client, cluster_name, interval) + IOLoop.current().add_timeout(start_time + 7, self.stop) + rate.increment() + self.wait(timeout=10) + end_time = time.time() + + Metric.StopMetricUpload(group_key) + Metric.QueryTimespan(self._client, group_key, start_time, end_time, partial(_OnQueryMetric, 2, 3)) + Metric.QueryTimespan(self._client, group_key, start_time + 4, end_time, partial(_OnQueryMetric, 1, 2)) + Metric.QueryTimespan(self._client, group_key, start_time, end_time - 4, partial(_OnQueryMetric, 1, 2)) + Metric.QueryTimespan(self._client, group_key, None, end_time, partial(_OnQueryMetric, 2, 3)) + Metric.QueryTimespan(self._client, group_key, start_time, None, partial(_OnQueryMetric, 2, 3)) + + # Setting both start_time and end_time to None fails. + self.assertRaises(AssertionError, + Metric.QueryTimespan, self._client, group_key, None, None, partial(_OnQueryMetric, 2, 3)) + + @async_test + def testMetricsAggregator(self): + """Test metrics aggregation with data from multiple machines""" + num_machines = 5 + num_samples = 15 + sample_duration = 60.0 + group_key = 'agg_test_group_key' + fake_time = 0 + managers = [] + + def fake_time_func(): + return fake_time + + def _OnAggregation(aggregator): + base_sum = sum(range(1, num_machines + 1)) + base_avg = base_sum / num_machines + + agg_total = aggregator.counter_data['aggtest.total'] + agg_delta = aggregator.counter_data['aggtest.delta'] + agg_rate = aggregator.counter_data['aggtest.rate'] + agg_avg = aggregator.counter_data['aggtest.avg'] + for s in range(num_samples): + # Check timestamp values + for cd in aggregator.counter_data.itervalues(): + self.assertEqual(cd.cluster_total[s][0], sample_duration * (s + 1)) + self.assertEqual(cd.cluster_avg[s][0], sample_duration * (s + 1)) + + # Check aggregate total + self.assertEqual(agg_total.cluster_total[s][1], base_sum * (s + 1)) + self.assertEqual(agg_total.cluster_avg[s][1], base_avg * (s + 1)) + + # Check aggregate delta + self.assertEqual(agg_delta.cluster_total[s][1], base_sum) + self.assertEqual(agg_delta.cluster_avg[s][1], base_avg) + + # Check aggregate rate + self.assertEqual(agg_rate.cluster_total[s][1], base_sum / sample_duration) + self.assertEqual(agg_rate.cluster_avg[s][1], base_avg / sample_duration) + + # Check aggregate avg + self.assertEqual(agg_avg.cluster_total[s][1], base_sum) + self.assertEqual(agg_avg.cluster_avg[s][1], base_avg) + + for m in range(1, num_machines + 1): + machine_name = 'machine%d' % m + + # Check per-machine total + mtotal = agg_total.machine_data[machine_name] + self.assertEqual(mtotal[s][1], m * (s + 1)) + + # Check per-machine delta + mdelta = agg_delta.machine_data[machine_name] + self.assertEqual(mdelta[s][1], m) + + # Check per-machine rate + mrate = agg_rate.machine_data[machine_name] + self.assertEqual(mrate[s][1], m / sample_duration) + + # Check per-machine avg + mavg = agg_avg.machine_data[machine_name] + self.assertEqual(mavg[s][1], m) + + self.stop() + + + def _OnMetricsUploaded(): + AggregatedMetric.CreateAggregateForTimespan(self._client, group_key, 0, sample_duration * num_samples, + managers[0], _OnAggregation) + + with util.Barrier(_OnMetricsUploaded) as b: + for m in range(1, num_machines + 1): + cm = counters._CounterManager() + cm.register(counters._TotalCounter('aggtest.total', 'Test Total')) + cm.register(counters._DeltaCounter('aggtest.delta', 'Test Delta')) + cm.register(counters._RateCounter('aggtest.rate', 'Test Rate', time_func=fake_time_func)) + cm.register(counters._AverageCounter('aggtest.avg', 'Test Average')) + + fake_time = 0 + meter = counters.Meter(cm) + managers.append(cm) + for s in range(num_samples): + cm.aggtest.total.increment(m) + cm.aggtest.delta.increment(m) + cm.aggtest.rate.increment(m) + cm.aggtest.avg.add(m) + cm.aggtest.avg.add(m) + fake_time += sample_duration + sample = json.dumps(meter.sample()) + metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample) + metric.Update(self._client, b.Callback()) diff --git a/backend/db/test/notification_test.py b/backend/db/test/notification_test.py new file mode 100644 index 0000000..ae348c0 --- /dev/null +++ b/backend/db/test/notification_test.py @@ -0,0 +1,60 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Notification & push notifications. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +import time + +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.op.op_context import EnterOpContext + +from base_test import DBBaseTestCase + + +class NotificationTestCase(DBBaseTestCase): + def testSimulateNotificationRaces(self): + """Try to create notification with same id twice to simulate race condition.""" + notification = Notification(self._user.user_id, 100) + notification.name = 'test' + notification.timestamp = time.time() + notification.sender_id = self._user.user_id + notification.sender_device_id = 1 + notification.badge = 0 + notification.activity_id = 'a123' + notification.viewpoint_id = 'v123' + + success = self._RunAsync(notification._TryUpdate, self._client) + self.assertTrue(success) + + notification.badge = 1 + success = self._RunAsync(notification._TryUpdate, self._client) + self.assertFalse(success) + + def testNotificationRaces(self): + """Concurrently create many notifications to force races.""" + op = Operation(1, 'o123') + with util.ArrayBarrier(self.stop) as b: + for i in xrange(10): + Notification.CreateForUser(self._client, + op, + 1, + 'test', + callback=b.Callback(), + invalidate={'invalid': True}, + activity_id='a123', + viewpoint_id='v%d' % i, + inc_badge=True) + notifications = self.wait() + + for i, notification in enumerate(notifications): + self.assertEqual(notification.user_id, 1) + self.assertEqual(notification.name, 'test') + self.assertEqual(notification.activity_id, 'a123') + self.assertEqual(notification.viewpoint_id, 'v%d' % i) + self.assertEqual(notification.badge, i + 1) diff --git a/backend/db/test/op_manager_test.py b/backend/db/test/op_manager_test.py new file mode 100644 index 0000000..a9f6aff --- /dev/null +++ b/backend/db/test/op_manager_test.py @@ -0,0 +1,526 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for OpManager and UserOpManager. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import json +import time + +from base_test import DBBaseTestCase +from datetime import timedelta +from tornado import gen +from viewfinder.backend.base.exceptions import CannotWaitError, PermissionError +from viewfinder.backend.db.base import util +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.op.op_manager import OpManager, OpMapEntry +from viewfinder.backend.op.user_op_manager import UserOpManager + + +class OpManagerTestCase(DBBaseTestCase): + def setUp(self): + super(OpManagerTestCase, self).setUp() + self._id = 1 + self._method_count = 0 + self._wait_count = 0 + + # Speed up retries and decrease limits for testing. + UserOpManager._INITIAL_BACKOFF_SECS = 0.05 + OpManager._MAX_SCAN_ABANDONED_LOCKS_INTERVAL = timedelta(seconds=0.5) + OpManager._MAX_SCAN_FAILED_OPS_INTERVAL = timedelta(seconds=0.5) + OpManager._SCAN_LIMIT = 1 + + # Remember the current OpManager instance in order to restore it after. + self.prev_op_mgr = OpManager.Instance() + + def tearDown(self): + # Restore original values. + OpManager.SetInstance(self.prev_op_mgr) + UserOpManager._INITIAL_BACKOFF_SECS = 8.0 + OpManager._MAX_SCAN_ABANDONED_LOCKS_INTERVAL = timedelta(seconds=60) + OpManager._MAX_SCAN_FAILED_OPS_INTERVAL = timedelta(hours=6) + OpManager._SCAN_LIMIT = 10 + super(OpManagerTestCase, self).tearDown() + + def testMultipleOps(self): + """Test multiple operations executed by OpManager.""" + def _OpMethod(client, callback): + self._method_count += 1 + callback() + if self._method_count == 3: + self.io_loop.add_callback(self.stop) + + def _OnWait(): + self._wait_count += 1 + + op_mgr = self._CreateOpManager(handlers=[_OpMethod]) + + op = self._CreateTestOp(user_id=1, handler=_OpMethod) + op_mgr.MaybeExecuteOp(self._client, op.user_id, op.operation_id, wait_callback=_OnWait) + + op = self._CreateTestOp(user_id=2, handler=_OpMethod) + op_mgr.MaybeExecuteOp(self._client, op.user_id, op.operation_id) + + op = self._CreateTestOp(user_id=1, handler=_OpMethod) + op_mgr.MaybeExecuteOp(self._client, op.user_id, op.operation_id) + + self.wait() + self.assertEqual(self._wait_count, 1) + + self._RunAsync(op_mgr.Drain) + + def testScanAbandonedLocks(self): + """Test scanning for locks which were abandoned due to server failure.""" + def _OpMethod(client, callback): + self._method_count += 1 + callback() + if self._method_count == 10: + self.io_loop.add_callback(self.stop) + + for i in xrange(10): + lock = self._AcquireOpLock(user_id=i / 2) + lock.Abandon(self._client, self.stop) + self.wait() + self._CreateTestOp(user_id=i / 2, handler=_OpMethod) + + # Add an expired non-op lock to ensure that it's skipped over during scan. + Lock.TryAcquire(self._client, LockResourceType.Job, 'v0', lambda lock, status: self.stop(lock), + detect_abandonment=True) + lock = self.wait() + lock.Abandon(self._client, self.stop) + self.wait() + + # Now scan for abandoned locks. + op_mgr = self._CreateOpManager(handlers=[_OpMethod]) + op_mgr._ScanAbandonedLocks() + self.wait() + + self._RunAsync(op_mgr.Drain) + + def testResourceData(self): + """Abandon an op lock with resource data set, and make sure that the op is run first.""" + def _OpMethod1(client, callback): + self._method_count += 1 + callback() + + def _OpMethod2(client, callback): + # Make sure that _OpMethod1 was already called. + self.assertEqual(self._method_count, 1) + callback() + self.io_loop.add_callback(self.stop) + + op = self._CreateTestOp(user_id=100, handler=_OpMethod2) + op = self._CreateTestOp(user_id=100, handler=_OpMethod1) + lock = self._AcquireOpLock(user_id=100, operation_id=op.operation_id) + self._RunAsync(lock.Abandon, self._client) + + op_mgr = self._CreateOpManager(handlers=[_OpMethod1, _OpMethod2]) + op_mgr._ScanAbandonedLocks() + self.wait() + + self._RunAsync(op_mgr.Drain) + + def testScanFailedOps(self): + """Test scanning for ops which have failed.""" + def _FlakyOpMethod(client, callback): + """Fails 8 times and then succeeds.""" + self._method_count += 1 + if self._method_count <= 8: + raise Exception('some transient failure') + callback() + self.io_loop.add_callback(self.stop) + + self._ExecuteOp(user_id=1, handler=_FlakyOpMethod, wait_for_op=False) + self._ExecuteOp(user_id=2, handler=_FlakyOpMethod, wait_for_op=False) + self.assertEqual(self._method_count, 6) + + op_mgr = self._CreateOpManager(handlers=[_FlakyOpMethod]) + op_mgr._ScanFailedOps() + self.wait() + + self._RunAsync(op_mgr.Drain) + + def testSimpleUserOp(self): + """Test simple operation that completes successfully.""" + self._ExecuteOp(user_id=1, handler=self._OpMethod) + self.assertEqual(self._method_count, 1) + + def testUserOpWithArgs(self): + """Test simple operation that completes successfully.""" + def _OpMethodWithArgs(client, callback, arg1, arg2): + assert arg1 == 'foo' and arg2 == 10 + self._method_count += 1 + callback() + + self._ExecuteOp(user_id=1, handler=_OpMethodWithArgs, arg1='foo', arg2=10) + self.assertEqual(self._method_count, 1) + + def testMultipleUserOps(self): + """Test multiple operations executed by UserOpManager.""" + self._CreateTestOp(user_id=1, handler=self._OpMethod) + self._CreateTestOp(user_id=1, handler=self._OpMethod) + self._CreateTestOp(user_id=2, handler=self._OpMethod) + self._ExecuteOp(user_id=1, handler=self._OpMethod) + self.assertEqual(self._method_count, 3) + + def testAddUserOpsDuring(self): + """Test new ops added during UserOpManager execution.""" + def _AddOpMethod3(client, callback): + """Create operation with lower op id.""" + with util.Barrier(callback) as b: + op_dict = self._CreateTestOpDict(user_id=1, handler=self._OpMethod) + op_dict['operation_id'] = Operation.ConstructOperationId(1, 1) + Operation.CreateFromKeywords(**op_dict).Update(client, b.Callback()) + + # Try to acquire lock, which has side effect of incrementing "acquire_failures" and triggering requery. + Lock.TryAcquire(self._client, LockResourceType.Operation, '1', b.Callback()) + + def _AddOpMethod2(client, callback): + """Create operation with lower op id.""" + op_dict = self._CreateTestOpDict(user_id=1, handler=_AddOpMethod3) + op_dict['operation_id'] = Operation.ConstructOperationId(1, 5) + Operation.CreateFromKeywords(**op_dict).Update(client, callback) + + # Explicitly call Execute for this op, since otherwise the UserOpManager won't "see" it, because + # it has an op-id that is lower than the currently executing op-id. + user_op_mgr.Execute(op_dict['operation_id']) + + def _AddOpMethod1(client, callback): + """Create operation with higher op id.""" + op_dict = self._CreateTestOpDict(user_id=1, handler=_AddOpMethod2) + Operation.CreateFromKeywords(**op_dict).Update(client, callback) + + self._id += 10 + op = self._CreateTestOp(user_id=1, handler=_AddOpMethod1) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_AddOpMethod1, _AddOpMethod2, + _AddOpMethod3, self._OpMethod], + callback=self.stop) + user_op_mgr.Execute(op.operation_id) + self.wait() + self.assertEqual(self._method_count, 1) + + def testLockFailure(self): + """Test case when UserOpManager cannot acquire lock.""" + # Acquire op lock for user #1. + self._AcquireOpLock(user_id=1) + + # Now try to execute op that also requires same lock. + self.assertRaises(CannotWaitError, self._ExecuteOp, user_id=1, handler=self._OpMethod) + + def testAbandonedLock(self): + """Test acquiring an abandoned lock.""" + lock = self._AcquireOpLock(user_id=1) + lock.Abandon(self._client, self.stop) + self.wait() + + self._ExecuteOp(user_id=1, handler=self._OpMethod) + self.assertEqual(self._method_count, 1) + + def testNoneOpId(self): + """Test operation_id=None given to UserOpManager.Execute.""" + self._CreateTestOp(user_id=1, handler=self._OpMethod) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[self._OpMethod], callback=self.stop) + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 1) + + def testUnknownOpId(self): + """Test unknown operation id given to UserOpManager.Execute.""" + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[self._OpMethod], callback=self.stop) + user_op_mgr.Execute(operation_id='unk1') + self.wait() + + def testMultipleExecuteCalls(self): + """Test multiple calls to UserOpManager.Execute.""" + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[self._OpMethod], callback=self.stop) + + self._CreateTestOp(user_id=1, handler=self._OpMethod) + user_op_mgr.Execute(operation_id='unk1') + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 1) + + self._CreateTestOp(user_id=1, handler=self._OpMethod) + user_op_mgr.Execute() + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 2) + + def testMultipleWaits(self): + """Test multiple UserOpManager.Execute, each with a wait callback for a different op.""" + with util.Barrier(self.stop) as b: + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[self._OpMethod], callback=b.Callback()) + + op1 = self._CreateTestOp(user_id=1, handler=self._OpMethod) + op2 = self._CreateTestOp(user_id=1, handler=self._OpMethod) + user_op_mgr.Execute(operation_id=op1.operation_id, wait_callback=b.Callback()) + user_op_mgr.Execute(operation_id=op1.operation_id, wait_callback=b.Callback()) + user_op_mgr.Execute(operation_id=op2.operation_id, wait_callback=b.Callback()) + + self.wait() + self.assertEqual(self._method_count, 2) + + def testWaitFailure(self): + """Test wait for operation that results in failure.""" + def _BuggyOpMethod(client, callback): + self._method_count += 1 + if self._method_count == 1: + raise Exception('some permanent failure') + callback() + + self.assertRaises(Exception, self._ExecuteOp, 1, _BuggyOpMethod) + self.assertEqual(self._method_count, 1) + + self._RunAsync(self.user_op_mgr.Drain) + + def testTransientFailure(self): + """Test op that fails once and then succeeds.""" + def _FlakyOpMethod(client, callback): + self._method_count += 1 + if self._method_count == 1: + raise Exception('some transient failure') + callback() + + self._ExecuteOp(user_id=1, handler=_FlakyOpMethod, wait_for_op=False) + self.assertEqual(self._method_count, 2) + + def testPermanentFailure(self): + """Test op that continually fails.""" + def _BuggyOpMethod(client, callback): + self._method_count += 1 + raise Exception('some permanent failure') + + op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop) + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 3) + + # Ensure that operation still exists in db with right values. + Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op)) + op = self.wait() + self.assertIsNotNone(op.first_failure) + self.assertIsNotNone(op.last_failure) + self.assertEqual(op.attempts, 3) + self.assertEqual(op.quarantine, 1) + self.assertGreater(op.backoff, 0) + + def testRerunBeforeContinue(self): + """Test that a previous operation is retried 3 times before the next operation is attempted.""" + @gen.coroutine + def _NextOpMethod(client): + # Multiply the method count, which tells us if this was called before or after the 2nd call + # to _FlakyOpMethod. + self._method_count *= 4 + + @gen.coroutine + def _FlakyOpMethod(client): + # Fail 1st to attempts, and succeed on the 3rd. + self._method_count += 1 + if self._method_count <= 2: + raise Exception('some transient failure') + + # Create two operations and ensure that first is re-run before second is completed. + op = self._CreateTestOp(user_id=1, handler=_FlakyOpMethod) + op2 = self._CreateTestOp(user_id=1, handler=_NextOpMethod) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_FlakyOpMethod, _NextOpMethod], callback=self.stop) + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 12) + + # Ensure that all operations completed and were deleted. + self.assertEqual(self._RunAsync(Operation.RangeQuery, self._client, 1, None, None, None), []) + + def testLargePermanentFailure(self): + """Test when DynamoDB limits the size of the traceback of a failing op.""" + def _BuggyOpMethod(client, callback, blob): + raise Exception('some permanent failure') + + op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod, blob='A' * (UserOpManager._MAX_OPERATION_SIZE - 200)) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop) + user_op_mgr.Execute() + self.wait() + + Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op)) + op = self.wait() + self.assertIsNotNone(op.first_failure) + self.assertIsNotNone(op.last_failure) + self.assertLessEqual(len(op.first_failure), 100) + self.assertLessEqual(len(op.last_failure), 100) + + def testAbortOfPermissionError(self): + """Test op that hits an abortable error.""" + def _BuggyOpMethod(client, callback): + self._method_count += 1 + # PermissionError is one of the exceptions which qualifies as abortable. + raise PermissionError('Not Authorized') + + op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop) + user_op_mgr.Execute() + self.wait() + self.assertEqual(self._method_count, 1) + + # Ensure that operation does not exist in the db. + Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op), must_exist=False) + op = self.wait() + self.assertTrue(op is None) + + def testMissingOpId(self): + """Provide non-existent op-id to UserOpManager.Execute.""" + self._CreateTestOp(user_id=1, handler=self._OpMethod) + user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[self._OpMethod], callback=self.stop) + user_op_mgr.Execute(operation_id='unknown') + self.wait() + self.assertEqual(self._method_count, 1) + + def testNestedOp(self): + """Test creation and invocation of nested operation.""" + @gen.coroutine + def _InnerMethod(client, arg1, arg2): + self._method_count += 1 + self.assertEqual(arg1, 1) + self.assertEqual(arg2, 'hello') + self.assertEqual(self._method_count, 2) + + # Assert that nested operation is derived from parent op. + inner_op = Operation.GetCurrent() + self.assertEqual(inner_op.user_id, outer_op.user_id) + self.assertEqual(inner_op.timestamp, outer_op.timestamp) + self.assertEqual(inner_op.operation_id, '+%s' % outer_op.operation_id) + + @gen.coroutine + def _OuterMethod(client): + self._method_count += 1 + if self._method_count == 1: + yield Operation.CreateNested(client, '_InnerMethod', {'arg1': 1, 'arg2': 'hello'}) + + # Create custom OpManager and make it the current instance for duration of test. + op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod]) + OpManager.SetInstance(op_mgr) + + outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod) + self._RunAsync(op_mgr.WaitForUserOps, self._client, 1) + self.assertEqual(self._method_count, 3) + + def testMultiNestedOp(self): + """Test nested op within nested op.""" + @gen.coroutine + def _InnererMethod(client, arg3): + self.assertTrue(Operation.GetCurrent().operation_id.startswith('++')) + self.assertEqual(arg3, 3) + self._method_count += 1 + + @gen.coroutine + def _InnerMethod(client, arg2): + self.assertEqual(arg2, 2) + self._method_count += 1 + if self._method_count == 2: + yield Operation.CreateNested(client, '_InnererMethod', {'arg3': 3}) + + @gen.coroutine + def _OuterMethod(client, arg1): + self.assertEqual(arg1, 1) + self._method_count += 1 + if self._method_count == 1: + yield Operation.CreateNested(client, '_InnerMethod', {'arg2': 2}) + + # Create custom OpManager and make it the current instance for duration of test. + op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod, _InnererMethod]) + OpManager.SetInstance(op_mgr) + + outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod, arg1=1) + self._RunAsync(op_mgr.WaitForUserOps, self._client, 1) + self.assertEqual(self._method_count, 5) + + def testNestedOpError(self): + """Test nested op that fails with errors.""" + @gen.coroutine + def _InnerMethod(client): + self._method_count += 1 + if self._method_count < 8: + raise Exception('permanent error') + + @gen.coroutine + def _OuterMethod(client): + self._method_count += 1 + if self._method_count < 8: + yield Operation.CreateNested(client, '_InnerMethod', {}) + self.assertEqual(Operation.GetCurrent().quarantine, 1) + + # Create custom OpManager and make it the current instance for duration of test. + op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod]) + OpManager.SetInstance(op_mgr) + + outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod) + op_mgr.MaybeExecuteOp(self._client, 1, None) + + # Now run failed ops (they should eventually succeed due to method_count < 8 checks) and ensure + # that ops complete. + while len(self._RunAsync(Operation.RangeQuery, self._client, 1, None, None, None)) != 0: + pass + + self.assertEqual(self._method_count, 9) + + def _OpMethod(self, client, callback): + self._method_count += 1 + callback() + + def _AcquireOpLock(self, user_id, operation_id=None): + Lock.TryAcquire(self._client, LockResourceType.Operation, str(user_id), lambda lock, status: self.stop(lock), + resource_data=operation_id) + return self.wait() + + def _ExecuteOp(self, user_id, handler, wait_for_op=True, **kwargs): + op = self._CreateTestOp(user_id=user_id, handler=handler, **kwargs) + + # Don't call self.stop() until we've gotten both the "completed all" and the "op wait" callbacks. + with util.Barrier(self.stop) as b: + self.user_op_mgr = self._CreateUserOpManager(user_id=user_id, handlers=[handler], callback=b.Callback()) + self.user_op_mgr.Execute(op.operation_id, b.Callback() if wait_for_op else None) + self.wait() + + # Ensure that lock is released. + lock_id = Lock.ConstructLockId(LockResourceType.Operation, str(user_id)) + Lock.Query(self._client, lock_id, None, lambda lock: self.stop(lock), must_exist=False) + lock = self.wait() + self.assertIsNone(lock, 'operation lock should have been released') + + return op + + def _CreateOpManager(self, handlers): + op_map = {handler.__name__: OpMapEntry(handler, []) for handler in handlers} + return OpManager(op_map, client=self._client, scan_ops=True) + + def _CreateUserOpManager(self, user_id, handlers, callback): + op_map = {handler.__name__: OpMapEntry(handler, []) for handler in handlers} + return UserOpManager(self._client, op_map, user_id, callback) + + def _CreateTestOp(self, user_id, handler, **kwargs): + Operation.ConstructOperationId(1, self._id) + self._id += 1 + + op_dict = self._CreateTestOpDict(user_id, handler, **kwargs) + op = Operation.CreateFromKeywords(**op_dict) + + op.Update(self._client, self.stop) + self.wait() + + return op + + def _CreateTestOpDict(self, user_id, handler, **kwargs): + op_id = Operation.ConstructOperationId(1, self._id) + self._id += 1 + + op_dict = {'user_id': user_id, + 'operation_id': op_id, + 'device_id': 1, + 'method': handler.__name__, + 'json': json.dumps(kwargs, indent=True), + 'timestamp': time.time(), + 'attempts': 0} + + return op_dict diff --git a/backend/db/test/operation_test.py b/backend/db/test/operation_test.py new file mode 100644 index 0000000..210cf70 --- /dev/null +++ b/backend/db/test/operation_test.py @@ -0,0 +1,244 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Tests for Operation and OpManager. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import time + +from functools import partial + +from viewfinder.backend.base import message, util, counters +from viewfinder.backend.base.exceptions import PermissionError +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.viewpoint import Viewpoint +from viewfinder.backend.op.op_manager import OpManager +from viewfinder.backend.op.user_op_manager import UserOpManager + +from base_test import DBBaseTestCase + +class OperationTestCase(DBBaseTestCase): + def setUp(self): + super(OperationTestCase, self).setUp() + # Speed up retries for testing. + UserOpManager._INITIAL_BACKOFF_SECS = 0.10 + UserOpManager._SMALL_INITIAL_BACKOFF_SECS = 0.10 + + self.meter = counters.Meter(counters.counters.viewfinder.operation) + self.meter_start = time.time() + + def tearDown(self): + super(OperationTestCase, self).tearDown() + UserOpManager._INITIAL_BACKOFF_SECS = 8.0 + + @async_test + def testDuplicateOperations(self): + """Verify that inserting duplicate operations is a no-op.""" + op_id = Operation.ConstructOperationId(self._mobile_dev.device_id, 100) + + with util.Barrier(self.stop) as b: + for i in xrange(10): + Operation.CreateAndExecute(self._client, self._user.user_id, self._mobile_dev.device_id, + 'HidePhotosOperation.Execute', + {'headers': {'op_id': op_id, 'op_timestamp': time.time(), + 'synchronous': True}, + 'user_id': self._user.user_id, + 'episodes': []}, b.Callback()) + + def testOperationRetries(self): + """Verify an operation is retried on failure.""" + # Acquire lock ahead of creation and execution of the operation. + # This will cause the operation to fail, but not be aborted, so it can be retried. + self._RunAsync(Lock.Acquire, self._client, LockResourceType.Viewpoint, self._user.private_vp_id, + Operation.ConstructOperationId(self._mobile_dev.device_id, 123)) + + # Make request to upload an episode + op = self._RunAsync(self._UploadPhotoOperation, self._user.user_id, self._mobile_dev.device_id, 1) + + start = time.time() + while True: + # Wait for op to execute. + self._RunAsync(self.io_loop.add_timeout, time.time() + UserOpManager._INITIAL_BACKOFF_SECS) + + op = self._RunAsync(Operation.Query, + self._client, + op.user_id, + op.operation_id, + None, + must_exist=False) + if op.attempts < 3: + self.assertTrue(not op.quarantine) + elif op.attempts == 3: + # After 3 attempts, op should go into quarantine. + self.assertTrue(op.first_failure is not None) + self.assertTrue(op.last_failure is not None) + self.assertEqual(op.quarantine, 1) + + # Manually goose the op manager to retry abandoned locks. + OpManager.Instance()._ScanFailedOps() + elif op.attempts == 4: + # No operations completed successfully. + self._CheckCounters(op.attempts, op.attempts - 1) + break + + @async_test + def testFailedRetriableOp(self): + """Verify that a failing retriable operation doesn't stop other ops from + continuing. + """ + def _OnQueryOrigOp(orig_op): + retry_count = orig_op.attempts - 1 + # 3 base operations expected plus 1 photo upload operation as part of _CreateBlockedOperation(). + self._CheckCounters(3 + 1 + retry_count, retry_count) + self.stop() + + def _OnSecondUploadOp(orig_op, op): + """Wait for the second operation and on completion, query the + original operation which is still failing. It should have a retry. + """ + Operation.WaitForOp(self._client, op.user_id, op.operation_id, + partial(Operation.Query, self._client, orig_op.user_id, + orig_op.operation_id, None, _OnQueryOrigOp)) + + def _OnFirstUpload(orig_op): + self._UploadPhotoOperation(orig_op.user_id, orig_op.device_id, 2, + partial(_OnSecondUploadOp, orig_op)) + + def _OnFirstUploadOp(orig_op, op): + Operation.WaitForOp(self._client, op.user_id, op.operation_id, + partial(_OnFirstUpload, orig_op)) + + def _OnCreateOp(orig_op): + """Set the operation's quarantine boolean to true and update.""" + orig_op.quarantine = 1 + orig_op.Update(self._client, partial(self._UploadPhotoOperation, + orig_op.user_id, orig_op.device_id, 1, + partial(_OnFirstUploadOp, orig_op))) + + self._CreateBlockedOperation(self._user.user_id, self._mobile_dev.device_id, _OnCreateOp) + + @async_test + def testFailedAbortableOp(self): + """Verify that a failing aborted operation doesn't stop other ops from + continuing. + """ + def _OnQueryOrigOp(orig_op): + # 3 base operations expected and no retries because the failed operation aborted. + self._CheckCounters(3, 0) + self.stop() + + def _OnSecondUploadOp(orig_op, op): + """Wait for the second operation and on completion, query the + original operation which is still failing. It should have a retry. + """ + Operation.WaitForOp(self._client, op.user_id, op.operation_id, + partial(_OnQueryOrigOp, orig_op)) + + def _OnFirstUpload(orig_op): + self._UploadPhotoOperation(orig_op.user_id, orig_op.device_id, 2, + partial(_OnSecondUploadOp, orig_op)) + + def _OnFirstUploadOp(orig_op, op): + Operation.WaitForOp(self._client, op.user_id, op.operation_id, + partial(_OnFirstUpload, orig_op)) + + def _OnCreateOp(orig_op): + self._UploadPhotoOperation(orig_op.user_id, orig_op.device_id, 1, + partial(_OnFirstUploadOp, orig_op)) + + self._CreateBadOperation(self._user.user_id, self._mobile_dev.device_id, _OnCreateOp) + + def testMismatchedDeviceId(self): + """ERROR: Try to create an op_id that does not match the client's + device. + """ + op_id = Operation.ConstructOperationId(100, 100) + + self.assertRaises(PermissionError, + self._RunAsync, + Operation.CreateAndExecute, + self._client, + self._user.user_id, + self._mobile_dev.device_id, + 'ShareNewOperation.Execute', + {'headers': {'op_id': op_id, 'op_timestamp': time.time(), + 'original_version': message.Message.ADD_OP_HEADER_VERSION}}) + + def _UploadPhotoOperation(self, user_id, device_id, seed, callback, photo_id=None): + """Creates an upload photos operation using seed to create unique ids.""" + request = {'user_id': user_id, + 'activity': {'activity_id': Activity.ConstructActivityId(time.time(), device_id, seed), + 'timestamp': time.time()}, + 'episode': {'user_id': user_id, + 'episode_id': Episode.ConstructEpisodeId(time.time(), device_id, seed), + 'timestamp': time.time()}, + 'photos': [{'photo_id': Photo.ConstructPhotoId(time.time(), device_id, seed) if photo_id is None else photo_id, + 'aspect_ratio': 1.3333, + 'timestamp': time.time(), + 'tn_size': 5 * 1024, + 'med_size': 50 * 1024, + 'full_size': 150 * 1024, + 'orig_size': 1200 * 1024}]} + Operation.CreateAndExecute(self._client, user_id, device_id, + 'UploadEpisodeOperation.Execute', request, callback) + + def _CreateBadOperation(self, user_id, device_id, callback): + """Creates a photo share for a photo which doesn't exist.""" + request = {'user_id': user_id, + 'activity': {'activity_id': 'a123', + 'timestamp': time.time()}, + 'viewpoint': {'viewpoint_id': Viewpoint.ConstructViewpointId(100, 100), + 'type': Viewpoint.EVENT}, + 'episodes': [{'existing_episode_id': 'eg8QVrk3S', + 'new_episode_id': 'eg8QVrk3T', + 'timestamp': time.time(), + 'photo_ids': ['pg8QVrk3S']}], + 'contacts': [{'identity': 'Local:testing1', + 'name': 'Peter Mattis'}]} + Operation.CreateAndExecute(self._client, user_id, device_id, + 'ShareNewOperation.Execute', request, callback) + + def _CreateBlockedOperation(self, user_id, device_id, callback): + """Creates a photo share after locking the viewpoint so that the operation will fail and get retried.""" + photo_id = Photo.ConstructPhotoId(time.time(), device_id, 123) + self._RunAsync(self._UploadPhotoOperation, user_id, device_id, 1, photo_id=photo_id) + + self._RunAsync(Lock.Acquire, self._client, LockResourceType.Viewpoint, 'vp123', + Operation.ConstructOperationId(device_id, 123)) + + request = {'user_id': user_id, + 'activity': {'activity_id': 'a123', + 'timestamp': time.time()}, + 'viewpoint': {'viewpoint_id': 'vp123', + 'type': Viewpoint.EVENT}, + 'episodes': [{'existing_episode_id': 'eg8QVrk3S', + 'new_episode_id': 'eg8QVrk3T', + 'timestamp': time.time(), + 'photo_ids': [photo_id]}], + 'contacts': [{'identity': 'Local:testing1', + 'name': 'Peter Mattis'}]} + + Operation.CreateAndExecute(self._client, user_id, device_id, + 'ShareNewOperation.Execute', request, callback) + + def _CheckCounters(self, expected_ops, expected_retries): + """Method used in a few tests to help verify performance counters.""" + sample = self.meter.sample() + elapsed = time.time() - self.meter_start + ops_per_min = (expected_ops / elapsed) * 60 + self.assertAlmostEqual(sample.viewfinder.operation.ops_per_min, ops_per_min, delta=ops_per_min * .1) + retries_per_min = (expected_retries / elapsed) * 60 + self.assertAlmostEqual(sample.viewfinder.operation.retries_per_min, retries_per_min, delta=retries_per_min * .1) + + # Assuming at least one op succeeded, avg_op_time should be > 0. + if expected_ops > expected_retries + 1: + self.assertGreater(sample.viewfinder.operation.avg_op_time, 0) + + self.meter_start += elapsed diff --git a/backend/db/test/photo_test.py b/backend/db/test/photo_test.py new file mode 100755 index 0000000..9d67316 --- /dev/null +++ b/backend/db/test/photo_test.py @@ -0,0 +1,84 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for Photo data object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import time + +from functools import partial + +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.photo import Photo + +from base_test import DBBaseTestCase + +class PhotoTestCase(DBBaseTestCase): + def testPhotoIdSortOrder(self): + """Create a series of photo ids and verify sort order. Sort order + is by time first (from newest to oldest), then from oldest to newest + device id, then from oldest to newest device photo id. + """ + # attributes are a tuple of time, device id, and device photo id. + photo_attributes = [(100, 1, 1), (100, 1, 2), (100, 1, 3), + (100, 2, 1), (100, 2, 2), (100, 2, 3), + (99, 1, 3), (99, 2, 2), (99, 3, 1), + (98, 3, 1), (97, 2, 1), (96, 1, 1)] + photo_ids = [Photo.ConstructPhotoId(p[0], p[1], p[2]) for p in photo_attributes] + self.assertEqual(photo_ids, sorted(photo_ids)) + + @async_test + def testQuery(self): + """Verify photo creation and query by photo id.""" + def _OnQuery(p, p2): + self.assertEqual(p2.caption, p.caption) + self.assertEqual(p2.photo_id, p.photo_id) + self.stop() + + def _OnCreatePhoto(p): + Photo.Query(self._client, p.photo_id, None, partial(_OnQuery, p)) + + photo_id = Photo.ConstructPhotoId(time.time(), self._mobile_dev.device_id, 1) + episode_id = Episode.ConstructEpisodeId(time.time(), self._mobile_dev.device_id, 2) + p_dict = {'photo_id': photo_id, + 'episode_id' : episode_id, + 'user_id': self._user.user_id, + 'caption': 'a photo'} + Photo.CreateNew(self._client, callback=_OnCreatePhoto, **p_dict) + + @async_test + def testUpdateAttribute(self): + """Verify update of a photo attribute.""" + def _OnUpdate(p): + p.aspect_ratio = None + p.Update(self._client, self.stop) + + def _OnQuery(p): + p.content_type = 'image/png' + p.Update(self._client, partial(_OnUpdate, p)) + + def _OnCreatePhoto(p): + Photo.Query(self._client, p.photo_id, None, _OnQuery) + + photo_id = Photo.ConstructPhotoId(time.time(), self._mobile_dev.device_id, 1) + episode_id = Episode.ConstructEpisodeId(time.time(), self._mobile_dev.device_id, 2) + p_dict = {'photo_id': photo_id, + 'episode_id' : episode_id, + 'user_id': self._user.user_id, + 'caption': 'A Photo'} + Photo.CreateNew(self._client, callback=_OnCreatePhoto, **p_dict) + + @async_test + def testMissing(self): + """Verify query for a missing photo fails.""" + def _OnQuery(p): + assert False, 'photo query should fail with missing key' + def _OnMissing(type, value, traceback): + self.stop() + return True + + with util.MonoBarrier(_OnQuery, on_exception=_OnMissing) as b: + Photo.Query(self._client, str(1L << 63), None, b.Callback()) diff --git a/backend/db/test/post_test.py b/backend/db/test/post_test.py new file mode 100644 index 0000000..fb8d221 --- /dev/null +++ b/backend/db/test/post_test.py @@ -0,0 +1,94 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for Post data object. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import time +import unittest + +from functools import partial + +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.episode import Episode +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post + +from base_test import DBBaseTestCase + +class PostTestCase(DBBaseTestCase): + def testPostIdConstruction(self): + """Verify round-trip of various post-ids.""" + def _RoundTripPostId(original_episode_id, original_photo_id): + post_id = Post.ConstructPostId(original_episode_id, original_photo_id) + new_episode_id, new_photo_id = Post.DeconstructPostId(post_id) + self.assertEqual(original_episode_id, new_episode_id) + self.assertEqual(original_photo_id, new_photo_id) + + _RoundTripPostId(Episode.ConstructEpisodeId(time.time(), 0, 0), + Photo.ConstructPhotoId(time.time(), 0, 0)) + + _RoundTripPostId(Episode.ConstructEpisodeId(time.time(), 1, (127, 'extra')), + Photo.ConstructPhotoId(time.time(), 1, (127, 'extra'))) + + _RoundTripPostId(Episode.ConstructEpisodeId(time.time(), 1, (128, None)), + Photo.ConstructPhotoId(time.time(), 1, (128, None))) + + _RoundTripPostId(Episode.ConstructEpisodeId(time.time(), 4000000000, (5000000000, 'v123')), + Photo.ConstructPhotoId(time.time(), 6000000000, (7000000000, 'v123'))) + + def testPostIdOrdering(self): + """Verify that post_id sorts like (episode_id, photo_id) does.""" + def _Compare(episode_id1, photo_id1, episode_id2, photo_id2): + result = cmp(episode_id1, episode_id2) + if result == 0: + result = cmp(photo_id1, photo_id2) + + post_id1 = Post.ConstructPostId(episode_id1, photo_id1) + post_id2 = Post.ConstructPostId(episode_id2, photo_id2) + self.assertEqual(cmp(post_id1, post_id2), result) + + timestamp = time.time() + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 1, (127, None)) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 1, (128, None)) + photo_id1 = Photo.ConstructPhotoId(timestamp, 1, 128) + photo_id2 = Photo.ConstructPhotoId(timestamp, 1, 127) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 127, 1) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 128, 1) + photo_id1 = Photo.ConstructPhotoId(timestamp, 128, (1, None)) + photo_id2 = Photo.ConstructPhotoId(timestamp, 127, (1, None)) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 0, 0) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 0, 0) + photo_id1 = Photo.ConstructPhotoId(timestamp, 0, 0) + photo_id2 = Photo.ConstructPhotoId(timestamp, 0, 0) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 1, 0) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 1, 1) + photo_id1 = Photo.ConstructPhotoId(timestamp, 1, 1) + photo_id2 = Photo.ConstructPhotoId(timestamp, 1, 0) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(0, 0, 0) + episode_id2 = Episode.ConstructEpisodeId(1, 0, 0) + photo_id1 = Photo.ConstructPhotoId(1, 0, (0, None)) + photo_id2 = Photo.ConstructPhotoId(0, 0, (0, None)) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 0, (0, '1')) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 0, (0, '2')) + photo_id1 = Photo.ConstructPhotoId(timestamp, 0, (0, None)) + photo_id2 = Photo.ConstructPhotoId(timestamp, 0, (0, None)) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) + + episode_id1 = Episode.ConstructEpisodeId(timestamp, 0, 0) + episode_id2 = Episode.ConstructEpisodeId(timestamp, 0, 0) + photo_id1 = Photo.ConstructPhotoId(timestamp, 0, (0, u'ab')) + photo_id2 = Photo.ConstructPhotoId(timestamp, 0, (0, u'cd')) + _Compare(episode_id1, photo_id1, episode_id2, photo_id2) diff --git a/backend/db/test/query_parser_test.py b/backend/db/test/query_parser_test.py new file mode 100644 index 0000000..d855b18 --- /dev/null +++ b/backend/db/test/query_parser_test.py @@ -0,0 +1,34 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Query parser tests. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball),' \ + 'mike@emailscrubbed.com (Mike Purtell)' + +import unittest + +from viewfinder.backend.db import query_parser, vf_schema + +class QueryParserTestCase(unittest.TestCase): + def TryQuery(self, query, param_dict=None): + query = query_parser.Query(vf_schema.SCHEMA, query) + query.PrintTree(param_dict) + + def testQueryParser(self): + self.TryQuery('user.given_name=Spencer') + self.TryQuery('user.given_name=Spencer & user.family_name=Kimball') + self.TryQuery('user.given_name=Spencer & user.family_name=Kimball' + ' - user.email="spencer@example.dot.com"') + self.TryQuery('photo.caption="a little bit of text a"') + self.TryQuery('photo.caption="search this text but not that" - ' + 'photo.caption="but not this phrase"') + + # now, test that parameterized queries work. + self.TryQuery('user.given_name={fn}', {'fn': 'Spencer'}) + self.TryQuery('user.given_name={fn} & user.family_name={ln}', {'fn': 'Spencer', 'ln': 'Kimball'}) + self.TryQuery('user.given_name={fn} & user.family_name={ln} - user.email={e}', + {'fn': 'Spencer', 'ln': 'Kimball', 'e': 'spencer@example.dot.com'}) + self.TryQuery('photo.caption={c}', {'c': 'a little bit of text a'}) + self.TryQuery('photo.caption={c1} - photo.caption={c2}', + {'c1': 'search this text but not that', 'c2': 'but not this phrase'}) diff --git a/backend/db/test/schema_test.py b/backend/db/test/schema_test.py new file mode 100644 index 0000000..3a9aec0 --- /dev/null +++ b/backend/db/test/schema_test.py @@ -0,0 +1,171 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Schema tests. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import json +import unittest + +from contextlib import contextmanager +from keyczar import errors, keyczar, keyinfo +from tornado import options +from viewfinder.backend.base import base_options # imported for option definitions +from viewfinder.backend.base import keyczar_dict, secrets +from viewfinder.backend.db.schema import Column, CryptColumn, _CryptValue +from viewfinder.backend.db.indexers import Indexer, SecondaryIndexer, FullTextIndexer + +class IndexerTestCase(unittest.TestCase): + def setUp(self): + self.col = Column('test', 'te', str) + + def testOptions(self): + """Test the optional indexer expansion settings.""" + value = 'one two three' + pos_single = [[0], [1], [2]] + pos_mphone = [[0], [1], [2], [2]] + self._VerifyIndex(FullTextIndexer(metaphone=Indexer.Option.NO), value, + ['te:one', 'te:two', 'te:three'], pos_single) + self._VerifyIndex(FullTextIndexer(metaphone=Indexer.Option.YES), value, + ['te:one', 'te:two', 'te:three', 'te:AN', 'te:T', 'te:0R', 'te:TR'], + pos_single + pos_mphone) + self._VerifyIndex(FullTextIndexer(metaphone=Indexer.Option.ONLY), value, + ['te:AN', 'te:T', 'te:0R', 'te:TR'], pos_mphone) + self._VerifyIndex(FullTextIndexer(metaphone=Indexer.Option.YES), 'food foot', + ['te:food', 'te:foot', 'te:FT'], [[0], [1], [0, 1]]) + + def testFullTextIndexer(self): + """Verifies operation of the full-text indexer, including stop words, + punctuation separation and position lists. + """ + tok = FullTextIndexer() + self._VerifyIndex(tok, 'one two three', ['te:one', 'te:two', 'te:three'], [[0], [1], [2]]) + self._VerifyIndex(tok, 'one-two.three', ['te:one', 'te:two', 'te:three'], [[0], [1], [2]]) + self._VerifyIndex(tok, 'one_two=three', ['te:one', 'te:two', 'te:three'], [[0], [1], [2]]) + self._VerifyIndex(tok, 'one t three', ['te:one', 'te:three'], [[0], [2]]) + self._VerifyIndex(tok, 'one one three', ['te:one', 'te:three'], [[0, 1], [2]]) + self._VerifyIndex(tok, "my, ain't it grand to have her to myself?", ["te:ain't", 'te:grand'], [[1], [3]]) + + def testQueryString(self): + """Verifies query string generation.""" + indexer = SecondaryIndexer() + self.assertEqual(indexer.GetQueryString(self.col, 'foo'), '"te:foo"') + tok = FullTextIndexer() + self.assertEqual(tok.GetQueryString(self.col, 'foo'), 'te:foo') + self.assertEqual(tok.GetQueryString(self.col, 'foo bar'), '(te:foo + te:bar)') + self.assertEqual(tok.GetQueryString(self.col, 'is foo = bar?'), '(_ + te:foo + te:bar)') + self.assertEqual(tok.GetQueryString(self.col, 'is foo equal to bar or is it not equal?'), + '(_ + te:foo + te:equal + _ + te:bar + _ + _ + _ + _ + te:equal)') + tok = FullTextIndexer(metaphone=Indexer.Option.YES) + self.assertEqual(tok.GetQueryString(self.col, 'foo'), '(te:foo | te:F)') + self.assertEqual(tok.GetQueryString(self.col, 'one or two or three'), + '((te:AN | te:one) + _ + (te:T | te:two) + _ + (te:TR | te:three | te:0R))') + + # Disable this test as it takes longer than is useful. + def TestLongPosition(self): + """Verifies position works up to 2^16 words and then is ignored after. + """ + tok = FullTextIndexer() + value = 'test ' * (1 << 16 + 1) + 'test2' + positions = [range(1 << 16), []] + self._VerifyIndex(tok, value, ['te:test', 'te:test2'], positions) + + def testSecondaryIndexer(self): + """Test secondary indexer emits column values.""" + indexer = SecondaryIndexer() + self._VerifyIndex(indexer, 'foo', ['te:foo'], None) + self._VerifyIndex(indexer, 'bar', ['te:bar'], None) + self._VerifyIndex(indexer, 'baz', ['te:baz'], None) + + def _VerifyIndex(self, tok, value, terms, freight=None): + term_dict = tok.Index(self.col, value) + self.assertEqual(set(terms), set(term_dict.keys())) + if freight: + assert len(terms) == len(freight) + for term, data in zip(terms, freight): + self.assertEqual(data, tok.UnpackFreight(self.col, term_dict[term])) + else: + for term in terms: + self.assertFalse(term_dict[term]) + + +class ColumnTestCase(unittest.TestCase): + def setUp(self): + options.options.domain = 'goviewfinder.com' + secrets.InitSecretsForTest() + self._crypt_inst = CryptColumn('foo', 'f').NewInstance() + + def testCryptColumn(self): + """Unit test the CryptColumn object.""" + def _Roundtrip(value): + # Set the value. + self._crypt_inst.Set(value) + self.assertTrue(value is None or self._crypt_inst.IsModified()) + + # Get the value as a _DelayedCrypt instance. + delayed_value = self._crypt_inst.Get() + if value is None: + self.assertIsNone(delayed_value) + else: + self.assertEqual(delayed_value, delayed_value) + self.assertNotEqual(delayed_value, None) + self.assertEqual(value, delayed_value.Decrypt()) + + # Get the value as a JSON-serializable dict. + value2 = self._crypt_inst.Get(asdict=True) + if value is None: + self.assertIsNone(value2) + else: + self.assertEqual(value2, {'__crypt__': delayed_value._encrypted_value}) + + # Set the value as a __crypt__ dict. + self._crypt_inst.SetModified(False) + self._crypt_inst.Set(value2) + self.assertFalse(self._crypt_inst.IsModified()) + + _Roundtrip(None) + _Roundtrip(1.23) + _Roundtrip('') + _Roundtrip('some value') + _Roundtrip('some value') + _Roundtrip(' \t\n\0#:&*01fjsbos\x100\x1234\x12345678 {') + _Roundtrip([]) + _Roundtrip([1, 2, -1, 0]) + _Roundtrip({'foo': 'str', 'bar': 1.23, 'baz': [1, 2], 'bat': {}}) + + def testDbKeyRotation(self): + """Verify that db_crypt key can be rotated.""" + @contextmanager + def _OverrideSecret(secret, secret_value): + try: + old_secret_value = secrets.GetSharedSecretsManager()._secrets[secret] + secrets.GetSharedSecretsManager()._secrets[secret] = secret_value + # Clear the cached crypter. + if hasattr(_CryptValue, '_crypter'): + del _CryptValue._crypter + yield + finally: + secrets.GetSharedSecretsManager()._secrets[secret] = old_secret_value + if hasattr(_CryptValue, '_crypter'): + del _CryptValue._crypter + + # Encrypt a value using the original key. + plaintext = 'quick brown fox' + self._crypt_inst.Set(plaintext) + + # Add a new key to the keyset and make it primary and ensure that plaintext can still be recovered. + writer = keyczar_dict.DictWriter(secrets.GetSharedSecretsManager()._secrets['db_crypt']) + czar = keyczar.GenericKeyczar(keyczar_dict.DictReader(writer.dict)) + czar.AddVersion(keyinfo.PRIMARY) + czar.Write(writer) + + with _OverrideSecret('db_crypt', json.dumps(writer.dict)): + self.assertEqual(self._crypt_inst.Get().Decrypt(), plaintext) + + # Now remove old key and verify that plaintext cannot be recovered. + czar.Demote(1) + czar.Revoke(1) + czar.Write(writer) + with _OverrideSecret('db_crypt', json.dumps(writer.dict)): + self.assertRaises(errors.KeyNotFoundError, self._crypt_inst.Get().Decrypt) diff --git a/backend/db/test/subscription_test.py b/backend/db/test/subscription_test.py new file mode 100644 index 0000000..42127ea --- /dev/null +++ b/backend/db/test/subscription_test.py @@ -0,0 +1,46 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +from viewfinder.backend.db.test.base_test import DBBaseTestCase +from viewfinder.backend.db.subscription import Subscription +from viewfinder.backend.services import itunes_store +from viewfinder.backend.services.test import itunes_store_test + +class SubscriptionTestCase(DBBaseTestCase): + def testCreateFromITunes(self): + # Create an initial subscription. + verify_response = itunes_store.VerifyResponse(itunes_store_test.kReceiptData, itunes_store_test.MakeNewResponse()) + Subscription.RecordITunesTransaction(self._client, self.stop, user_id=self._user.user_id, verify_response=verify_response) + self.wait() + + # The test subscription is expired, so it won't be returned by default. + Subscription.QueryByUser(self._client, self.stop, self._user.user_id) + subs = self.wait() + self.assertEqual(len(subs), 0) + + # It's there with include_expired=True. + Subscription.QueryByUser(self._client, self.stop, self._user.user_id, include_expired=True) + subs = self.wait() + self.assertEqual(len(subs), 1) + self.assertEqual(subs[0].expiration_ts, itunes_store_test.kExpirationTime) + first_transaction_id = subs[0].transaction_id + + # Now add a renewal. + verify_response = itunes_store.VerifyResponse(itunes_store_test.kReceiptData, itunes_store_test.MakeRenewedResponse()) + Subscription.RecordITunesTransaction(self._client, self.stop, user_id=self._user.user_id, verify_response=verify_response) + self.wait() + + # Only the latest transaction is returned. + Subscription.QueryByUser(self._client, self.stop, self._user.user_id, include_expired=True) + subs = self.wait() + self.assertEqual(len(subs), 1) + self.assertEqual(subs[0].expiration_ts, itunes_store_test.kExpirationTime2) + self.assertNotEqual(subs[0].transaction_id, first_transaction_id) + + # With include_history=True we get both transactions. + Subscription.QueryByUser(self._client, self.stop, self._user.user_id, include_history=True) + subs = self.wait() + self.assertEqual(len(subs), 2) + self.assertItemsEqual([sub.expiration_ts for sub in subs], + [itunes_store_test.kExpirationTime, itunes_store_test.kExpirationTime2]) diff --git a/backend/db/test/test_rename.py b/backend/db/test/test_rename.py new file mode 100644 index 0000000..4abc1e7 --- /dev/null +++ b/backend/db/test/test_rename.py @@ -0,0 +1,17 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Test object for the TEST_RENAME database testing table. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + +@DBObject.map_table_attributes +class TestRename(DBRangeObject): + """Used for testing.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.TEST_RENAME) diff --git a/backend/db/test/user_photo_test.py b/backend/db/test/user_photo_test.py new file mode 100644 index 0000000..62e5b2d --- /dev/null +++ b/backend/db/test/user_photo_test.py @@ -0,0 +1,44 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved +"""Tests for UserPhoto data object.""" + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +from base_test import DBBaseTestCase + +from viewfinder.backend.db.user_photo import UserPhoto +from viewfinder.backend.db import versions + +class UserPhotoTestCase(DBBaseTestCase): + def testAssetKeyToFingerprint(self): + def test(asset_key, expected): + self.assertEqual(UserPhoto.AssetKeyToFingerprint(asset_key), expected) + test('', None) + test('a/', None) + test('a/b', None) + test('a/#', None) + test('a/b#', None) + test('a/b#c', 'a/#c') + test('a/b##c', 'a/#c') + test('a/assets-library://asset/asset.JPG?id=D31F1D3C-CFB7-458F-BACD-7862D72098A6&ext=JPG#e5ad400c2214088928ef8400dcfb87bb3059b742', + 'a/#e5ad400c2214088928ef8400dcfb87bb3059b742') + test('a/assets-library://asset/asset.JPG?id=D31F1D3C-CFB7-458F-BACD-7862D72098A6&ext=JPG', + None) + test('a/#e5ad400c2214088928ef8400dcfb87bb3059b742', + 'a/#e5ad400c2214088928ef8400dcfb87bb3059b742') + + def testMigration(self): + user_photo = UserPhoto.CreateFromKeywords( + user_id=1, photo_id='p1', + asset_keys=['a/b', 'a/c#d', 'a/e#d', 'a/f#g']) + user_photo._version = versions.REMOVE_ASSET_URLS.rank - 1 + self._RunAsync(versions.Version.MaybeMigrate, self._client, user_photo, + [versions.REMOVE_ASSET_URLS]) + print user_photo + self.assertEqual(user_photo.asset_keys.combine(), set(['a/#d', 'a/#g'])) + + def testMergeAssetKeys(self): + user_photo = UserPhoto.CreateFromKeywords( + user_id=1, photo_id='p1', + asset_keys=['a/#b', 'a/#f']) + user_photo.MergeAssetKeys(['a/b#c', 'a/d#c', 'a/e#f']) + self.assertEqual(user_photo.asset_keys.combine(), set(['a/#b', 'a/#c', 'a/#f'])) diff --git a/backend/db/test/user_test.py b/backend/db/test/user_test.py new file mode 100755 index 0000000..02cc49b --- /dev/null +++ b/backend/db/test/user_test.py @@ -0,0 +1,190 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Tests for User data object. +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import sys +import time +import unittest + +from functools import partial +from tornado import stack_context +from viewfinder.backend.base import util +from viewfinder.backend.base.testing import async_test +from viewfinder.backend.db.schema import Location +from viewfinder.backend.db import device, viewpoint +from viewfinder.backend.db.user import User + +from base_test import DBBaseTestCase + +class UserTestCase(DBBaseTestCase): + def _CreateDefaultUser(self): + return self._CreateUserAndDevice(user_dict={'user_id': 4, + 'given_name': 'Spencer', + 'family_name': 'Kimball', + 'email': 'spencer@emailscrubbed.com'}, + ident_dict={'key': 'Email:spencer@emailscrubbed.com', + 'authority': 'Test'}, + device_dict=None) + + def testPartialCreate(self): + """Creates a user with just a given name.""" + user_dict = {'user_id': 5, 'given_name': 'Spencer'} + user, device = self._CreateUserAndDevice(user_dict=user_dict, + ident_dict={'key': 'Email:spencer.kimball@foo.com', + 'authority': 'Test'}, + device_dict=None) + user._version = None + user.signing_key = None + user_dict['labels'] = [User.REGISTERED] + user_dict['email'] = 'spencer.kimball@foo.com' + user_dict['private_vp_id'] = 'v-k-' + user_dict['webapp_dev_id'] = 3 + user_dict['asset_id_seq'] = 1 + self.assertEqual(user._asdict(), user_dict) + + def testRegister(self): + """Creates a user via an OAUTH user dictionary.""" + u, dev = self._CreateUserAndDevice(user_dict={'user_id': 4, + 'given_name': 'Spencer', + 'family_name': 'Kimball', + 'locale': 'en:US'}, + ident_dict={'key': 'Local:0_0.1', + 'authority': 'Test'}, + device_dict={'device_id': 20, + 'version': 'alpha-1.0', + 'os': 'iOS 5.0.1', + 'platform': 'iPhone 4S', + 'country': 'US', + 'language': 'en'}) + + self.assertEqual(u.given_name, 'Spencer') + self.assertEqual(u.family_name, 'Kimball') + self.assertEqual(u.locale, 'en:US') + + self.assertTrue(dev.device_id > 0) + self.assertTrue(dev.version, 'alpha-1.0') + self.assertTrue(dev.os, 'iOS 5.0.1') + self.assertTrue(dev.platform, 'iPhone 4S') + self.assertTrue(dev.country, 'US') + self.assertTrue(dev.language, 'en') + + # Try another registration with the same identity, but some different data and a different device. + u2, dev2 = self._CreateUserAndDevice(user_dict={'user_id': 4, + 'given_name': 'Brian', + 'email': 'spencer@emailscrubbed.com'}, + ident_dict={'key': 'Local:0_0.1', + 'authority': 'Test'}, + device_dict={'device_id': 21, + 'version': 'alpha-1.0', + 'os': 'Android 4.0.3', + 'platform': 'Samsung Galaxy S'}) + + # On a second registration, the user id shouldn't change, new information should be added, but + # changed information should be ignored. + self.assertEqual(u.user_id, u2.user_id) + self.assertEqual(u2.given_name, 'Spencer') + self.assertEqual(u2.email, 'spencer@emailscrubbed.com') + + self.assertTrue(dev.device_id != dev2.device_id) + self.assertEqual(dev2.version, 'alpha-1.0') + self.assertEqual(dev2.os, 'Android 4.0.3') + self.assertEqual(dev2.platform, 'Samsung Galaxy S') + + # Try a registration with a different identity, same user. Use the original device, but change + # the app version number. + u3, dev3 = self._CreateUserAndDevice(user_dict={'user_id': u2.user_id, + 'link': 'http://www.goviewfinder.com'}, + ident_dict={'key': 'Local:0_0.2', + 'authority': 'Test'}, + device_dict={'device_id': dev.device_id, + 'version': 'alpha-1.2'}) + + self.assertEqual(u.user_id, u3.user_id) + self.assertEqual(u3.link, 'http://www.goviewfinder.com') + self.assertEqual(dev3.device_id, dev.device_id) + self.assertEqual(dev3.os, dev.os) + self.assertEqual(dev3.version, 'alpha-1.2') + + # Try a registration with an already-used identity. + try: + self._CreateUserAndDevice(user_dict={'user_id': 1000}, + ident_dict={'key': 'Local:0_0.1', + 'authority': 'Test'}, + device_dict={'device_id': dev3.device_id}) + assert False, 'third identity should have failed with already-in-use' + except Exception as e: + assert e.message.startswith('the identity is already in use'), e + identities = self._RunAsync(u3.QueryIdentities, self._client) + keys = [i.key for i in identities] + self.assertTrue('Local:0_0.1' in keys) + self.assertTrue('Local:0_0.2' in keys) + + def testRegisterViaWebApp(self): + """Register from web application.""" + u, dev = self._CreateUserAndDevice(user_dict={'user_id': 4, + 'name': 'Spencer Kimball'}, + ident_dict={'key': 'Test:1', + 'authority': 'Test'}, + device_dict=None) + self.assertEqual(u.name, 'Spencer Kimball') + self.assertTrue(u.webapp_dev_id > 0) + self.assertIsNone(dev) + + def testAddDevice(self): + """Register with no mobile device, then add one.""" + u, dev = self._CreateUserAndDevice(user_dict={'user_id': 4, + 'name': 'Spencer Kimball'}, + ident_dict={'key': 'Local:1', + 'authority': 'Test'}, + device_dict=None) + + u2, dev2 = self._CreateUserAndDevice(user_dict={'user_id': 4}, + ident_dict={'key': 'Local:1', + 'authority': 'Test'}, + device_dict={'device_id': 30}) + + self.assertEqual(u2.user_id, u.user_id) + self.assertIsNone(dev) + self.assertIsNotNone(dev2) + + def testQueryUpdate(self): + """Update a user.""" + u, d = self._CreateDefaultUser() + + # Now update the user. + u2, d2 = self._CreateUserAndDevice(user_dict={'user_id': u.user_id, + 'email': 'spencer.kimball@emailscrubbed.com'}, + ident_dict={'key': 'Email:spencer@emailscrubbed.com', + 'authority': 'Facebook'}, + device_dict=None) + self.assertEqual(u.email, u2.email) + + def testAllocateAssetIds(self): + """Verify the per-device allocation of user ids.""" + self._RunAsync(User.AllocateAssetIds, self._client, self._user.user_id, 5) + user = self._RunAsync(User.Query, self._client, self._user.user_id, None) + self.assertEqual(user.asset_id_seq, 6) + + self._RunAsync(User.AllocateAssetIds, self._client, self._user.user_id, 100) + user = self._RunAsync(User.Query, self._client, self._user.user_id, None) + self.assertEqual(user.asset_id_seq, 106) + + def testPartialQuery(self): + """Test query of partial row data.""" + u, dev = self._CreateDefaultUser() + u2 = self._RunAsync(User.Query, self._client, u.user_id, ['given_name', 'family_name']) + self.assertEqual(u2.email, None) + self.assertEqual(u2.given_name, u.given_name) + self.assertEqual(u2.family_name, u.family_name) + + def testMissing(self): + """Test query of a non-existent user.""" + try: + self._RunAsync(User.Query, self._client, 1L << 63, None) + assert False, 'user query should fail with missing key' + except Exception as e: + pass diff --git a/backend/db/test/viewpoint_test.py b/backend/db/test/viewpoint_test.py new file mode 100644 index 0000000..d7f0edd --- /dev/null +++ b/backend/db/test/viewpoint_test.py @@ -0,0 +1,14 @@ +# Copyright 2013 Viewfinder Inc. All rights reserved. +"""Tests for Viewpoint data object.""" + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +from viewfinder.backend.db.viewpoint import Viewpoint + +from base_test import DBBaseTestCase + +class ViewpointTestCase(DBBaseTestCase): + def testRepr(self): + vp = Viewpoint.CreateFromKeywords(viewpoint_id='vp1', title='hello') + self.assertIn('vp1', repr(vp)) + self.assertNotIn('hello', repr(vp)) diff --git a/backend/db/tools/__init__.py b/backend/db/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/db/tools/admin_tool.py b/backend/db/tools/admin_tool.py new file mode 100644 index 0000000..4c469ec --- /dev/null +++ b/backend/db/tools/admin_tool.py @@ -0,0 +1,54 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Manage entries in the AdminPermissions table. + +Usage: + +# Add/modify a user as root. +python admin_tool.py --op=set --user= --rights=root +# Add/modify a user as support. +python admin_tool.py --op=set --user= --rights=support +# Add/modify a user with no rights. +python admin_tool.py --op=set --user= +# Delete a user. +python admin_tool.py --op=del --user= + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import logging +import sys + +from tornado import options +from viewfinder.backend.base import main +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.admin_permissions import AdminPermissions + +options.define('user', default=None, help='user to set/delete') +options.define('rights', default=None, multiple=True, help='list of rights ("root", "support"). Omit to clear.') +options.define('op', default=None, help='command: set, del') + +def ProcessAdmins(callback): + client = db_client.DBClient.Instance() + + assert options.options.user is not None + assert options.options.op is not None + + if options.options.op == 'set': + permissions = AdminPermissions(options.options.user, options.options.rights) + logging.info('committing %r' % permissions) + permissions.Update(client, callback) + + elif options.options.op == 'del': + admin = AdminPermissions(options.options.user) + logging.info('deleting %r' % admin) + admin.Delete(client, callback) + + else: + logging.error('unknown op: %s' % options.options.op) + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(ProcessAdmins)) diff --git a/backend/db/tools/clean_index_table.py b/backend/db/tools/clean_index_table.py new file mode 100644 index 0000000..de11f0b --- /dev/null +++ b/backend/db/tools/clean_index_table.py @@ -0,0 +1,75 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utility to scan and cleanup the index table. +Usage: +# Look for all entries with index terms for old tables: +python -m viewfinder.backend.db.tools.clean_index_table --hash_key_prefixes='df:,la:,me:' + +# Delete the entries. +python -m viewfinder.backend.db.tools.clean_index_table --hash_key_prefixes='df:,la:,me:' --delete=True + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import logging +import sys + +from tornado import gen, options +from viewfinder.backend.base import main +from viewfinder.backend.db import db_client, vf_schema + +options.define('hash_key_prefixes', default=[], multiple=True, + help='hash_key prefixes to delete.') +options.define('delete', default=False, + help='Actually delete entries.') + +@gen.engine +def Scan(client, table, callback): + """Scans an entire table. + """ + found_entries = {} + for prefix in options.options.hash_key_prefixes: + found_entries[prefix] = 0 + deleted = 0 + last_key = None + count = 0 + while True: + result = yield gen.Task(client.Scan, table.name, attributes=None, limit=50, excl_start_key=last_key) + count += len(result.items) + + for item in result.items: + value = item.get('t', None) + sort_key = item.get('k', None) + if value is None or sort_key is None: + continue + for prefix in options.options.hash_key_prefixes: + if value.startswith(prefix): + logging.info('matching item: %r' % item) + found_entries[prefix] += 1 + if options.options.delete: + logging.info('deleting item: %r' % item) + yield gen.Task(client.DeleteItem, table=table.name, key=db_client.DBKey(value, sort_key)) + deleted += 1 + if result.last_key: + last_key = result.last_key + else: + break + + logging.info('Found entries: %r' % found_entries) + logging.info('scanned %d items, deleted %d' % (count, deleted)) + callback() + +@gen.engine +def RunDBA(callback): + yield gen.Task(db_client.InitDB, vf_schema.SCHEMA, verify_or_create=True) + + table = vf_schema.SCHEMA.GetTable('Index') + client = db_client.DBClient.Instance() + + yield gen.Task(Scan, client, table) + + callback() + +if __name__ == '__main__': + sys.exit(main.InitAndRun(RunDBA, init_db=False)) diff --git a/backend/db/tools/dba.py b/backend/db/tools/dba.py new file mode 100644 index 0000000..32bae8c --- /dev/null +++ b/backend/db/tools/dba.py @@ -0,0 +1,149 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utility to manage high-level data objects. This is in +contrast to dba.py, which directly accesses and mutates +DynamoDB rows. Direct mutation is dangerous as it skirts +the layers observed by the Python backend/db/* classes, +which properly update secondary indexes and auxillary +tables such as Notification. + +See raw_dba.py for notes on quoting keys and attributes. + + +Usage: + +--op specifies the operation: + + SIMPLE OPERATIONS: + + - query: query a range for a table + --hash_key= [--start_key=] + - update: update an item in a table + --hash_key= [--range_key= + + COMPLEX OPERATIONS: + + - None + + +python -m viewfinder.backend.db.tools.dba --op= [--table=
] \ + [--hash_key=] [--range_key=] +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import pprint +import sys + +from functools import partial +from tornado import options +from viewfinder.backend.base import main +from viewfinder.backend.db import db_client, db_import, vf_schema +from viewfinder.backend.db.tools.util import AttrParser + +options.define('table', default=None, help='table name') +options.define('op', default=None, help='operation on database entities') +options.define('user_id', default=None, help='user id for operation, if applicable') +options.define('hash_key', default=None, help='primary hash key for item-level ops') +options.define('range_key', default=None, help='primary range key for item-level ops') +options.define('start_key', default=None, help='start range key for query') +options.define('end_key', default=None, help='end range key for query') +options.define('limit', default=None, help='maximum row limit on queries') +options.define('attributes', default=None, help='value attributes pairs (attr0=value0,attr1=value1,...) ' + 'can be quoted strings. Values are eval\'d') +options.define('col_names', default=None, multiple=True, help='column names to print when querying items') + + +def QueryRange(client, table, cls, key, range_desc, callback): + """Queries the contents of a range identified by --hash_key. + """ + def _OnQuery(retry_cb, count, items): + for item in items: + pprint.pprint(item) + if len(items) == 100: + retry_cb(items[-1].GetKey(), count + len(items)) + else: + return callback('queried %d items' % count) + + def _Query(last_key=None, count=0): + cls.RangeQuery(client, key.hash_key, range_desc, limit=100, + col_names=(options.options.col_names or None), + callback=partial(_OnQuery, _Query, count), + excl_start_key=last_key) + + _Query() + + +def RunDBA(callback): + """Runs op on each table listed in --tables.""" + client = db_client.DBClient.Instance() + op = options.options.op + + table = None + if options.options.table: + table = vf_schema.SCHEMA.GetTable(options.options.table) + assert table, 'unrecognized table name: %s' % options.options.table + cls = db_import.GetTableClass(table.name) + + key = None + if options.options.hash_key and options.options.range_key: + assert table.range_key_col + key = db_client.DBKey(eval(options.options.hash_key), eval(options.options.range_key)) + elif options.options.hash_key: + assert not table.range_key_col + key = db_client.DBKey(eval(options.options.hash_key), None) + + start_key = eval(options.options.start_key) if options.options.start_key else None + end_key = eval(options.options.end_key) if options.options.end_key else None + range_desc = None + if start_key and end_key: + range_desc = db_client.RangeOperator([start_key, end_key], 'BETWEEN') + elif start_key: + range_desc = db_client.RangeOperator([start_key], 'GT') + elif end_key: + range_desc = db_client.RangeOperator([end_key], 'LT') + + user_id = None + if options.options.user_id: + user_id = eval(options.options.user_id) + + limit = None + if options.options.limit: + limit = eval(options.options.limit) + + if options.options.attributes: + parser = AttrParser(table) + attrs = parser.Run(options.options.attributes) + if table and key: + attrs[table.hash_key_col.name] = key.hash_key + if key.range_key: + attrs[table.range_key_col.name] = key.range_key + logging.info('attributes: %s' % attrs) + + def _OnOp(*args, **kwargs): + if args: + logging.info('positional result args: %s' % pprint.pformat(args)) + if kwargs: + logging.info('keyword result args: %s' % pprint.pformat(kwargs)) + callback() + + if op in ('query', 'update'): + assert table, 'no table name specified for operation' + if op in ('query'): + assert table.range_key_col, 'Table %s is not composite' % table.name + + # Run the operation + logging.info('executing %s' % op) + if op == 'query': + QueryRange(client, table, cls, key, range_desc, _OnOp) + elif op == 'update': + o = cls() + o.UpdateFromKeywords(**attrs) + o.Update(client, _OnOp) + else: + raise Exception('unrecognized op: %s' % op) + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(RunDBA)) diff --git a/backend/db/tools/dbchk.py b/backend/db/tools/dbchk.py new file mode 100644 index 0000000..96aee91 --- /dev/null +++ b/backend/db/tools/dbchk.py @@ -0,0 +1,1110 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Crawls over the database, looking for problems and inconsistencies that +may have crept in due to bugs or lack of strong DB consistency. When problems +are found, logs them and e-mails notifications about them. Also repairs +problems if the --repair option is specified. + +Usage: + # Check the database, logging and e-mailing any corruptions found. + python dbchk.py + + # Lookup last successful status and only scan viewpoints updated after the previous run's start time. + # Additionally, skip run if the previous successful run started less than 6 hours ago. + python dbchk.py --smart_scan --hours_between_runs=6 + + # Repair corruptions found in the database during the check phase. + python dbchk.py --repair=True --viewpoints=vp1,vp2 +""" + +__author__ = 'andy@emailscrubbed.com (Andrew Kimball)' + +import json +import logging +import sys +import traceback +import time + +from collections import defaultdict +from copy import deepcopy +from functools import partial +from tornado import gen, options, stack_context +from tornado.ioloop import IOLoop +from viewfinder.backend.base import constants, main, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.accounting import Accounting +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.comment import Comment +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.followed import Followed +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.job import Job +from viewfinder.backend.db.notification import Notification +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.photo import Photo +from viewfinder.backend.db.post import Post +from viewfinder.backend.db.user_post import UserPost +from viewfinder.backend.db.viewpoint import Viewpoint +from viewfinder.backend.op.notification_manager import NotificationManager +from viewfinder.backend.op.op_context import EnterOpContext +from viewfinder.backend.services.email_mgr import EmailManager, SendGridEmailManager + + +options.define('viewpoints', type=str, default=[], multiple=True, + help='check (and possibly repair) this list of viewpoint ids, ignores smart_scan; ' + 'check all viewpoints if None') + +options.define('smart_scan', type=bool, default=False, + help='only scan viewpoints updated since the last dbchk run') + +options.define('repair', type=bool, default=False, + help='if true, automatically repair corruption in checked viewpoints') + +options.define('email', default='dbchk@emailscrubbed.com', + help='email to which to send corruption report; no report sent if set to blank') + +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:dbchk lock before running. Exit if acquire fails.') + +options.define('hours_between_runs', type=int, default=0, + help='minimum time since start of last successful full run (without --viewpoints)') + + +_TEST_MODE = False +"""If true, run the checker in test mode.""" + +_CHECK_THRESHOLD_TIMESPAN = constants.SECONDS_PER_HOUR +"""Don't check viewpoints that have been recently modified (within this +time window). +""" + + +class DatabaseChecker(object): + """Collection of methods that scan the database, looking for instances + of corruption. + """ + _MAX_EMAIL_INTERVAL = 60 * 30 + """Send notification email at most once every thirty minutes.""" + + _EMAIL_TEXT = 'Found corruption(s) in database:\n\n%s' + """Text to send in email reporting the corruption.""" + + _MAX_VISITED_USERS = 10000 + """When the visited_users set exceeds this number, trigger per-user actions for each user.""" + + def __init__(self, client, repair=False): + # Save DB client and repair flag. + self._client = client + self._repair = repair + + # Dictionary mapping viewpoint ids to list of detected corruptions in each viewpoint. + self._corruptions = {} + + # Set of users that are followers of visited viewpoints. When it reaches a certain level, + # we process each user in turn. We store the latest viewpoint that cause the visit so that + # we know which viewpoint to process to repair errors. + # Only followers of shared viewpoints are added since the per-user actions do not take + # default viewpoint information into account. + self._visited_users = {} + + # Reset time that last email was sent. + self._last_email = util.GetCurrentTimestamp() + + self._num_visited_viewpoints = 0 + + # Current viewpoint or user being processed, used in failed job message. + self._current_viewpoint = '' + self._current_user = '' + + @gen.engine + def CheckAllViewpoints(self, callback, last_scan=None): + """Scans the entire Viewpoint table, looking for corruption in each + viewpoint that has not already been scanned in a previous pass. + """ + # Clear email args used for testing. + self._email_args = None + + # Only scan for newly updated viewpoints if previous scan has taken place. + if last_scan is None: + scan_filter = None + else: + scan_filter = {'last_updated': db_client.ScanFilter([last_scan], 'GE')} + + yield gen.Task(self._ThrottledScan, + Viewpoint, + visitor=self.CheckViewpoint, + scan_filter=scan_filter, + max_read_units=Viewpoint._table.read_units) + + # Force a check of all visited users, we may have some remaining. + yield gen.Task(self._CheckVisitedUsers) + + # Check to see whether a corruption report needs to be emailed. + yield gen.Task(self._SendEmail) + + callback() + + @gen.engine + def CheckViewpointList(self, viewpoint_ids, callback): + """Looks for corruption in each of the viewpoints in the + "viewpoint_ids" list. + """ + # Clear email args used for testing. + self._email_args = None + + for vp_id in viewpoint_ids: + viewpoint = yield gen.Task(Viewpoint.Query, self._client, vp_id, None) + yield gen.Task(self.CheckViewpoint, viewpoint) + + # Force a check of all visited users, we may have some remaining. + yield gen.Task(self._CheckVisitedUsers) + + # Check to see whether a corruption report needs to be emailed. + yield gen.Task(self._SendEmail) + + callback() + + @gen.engine + def CheckViewpoint(self, viewpoint, callback): + """Checks the specified viewpoint for various kinds of corruption.""" + # Don't check viewpoints that were modified within last hour, as operation might still be in progress. + if _TEST_MODE or viewpoint.last_updated <= time.time() - _CHECK_THRESHOLD_TIMESPAN: + logging.info('Processing viewpoint "%s"...' % viewpoint.viewpoint_id) + self._current_viewpoint = viewpoint.viewpoint_id + self._num_visited_viewpoints += 1 + + # Gather followers. + query_func = partial(Viewpoint.QueryFollowerIds, self._client, viewpoint.viewpoint_id) + follower_ids = yield gen.Task(self._CacheQuery, query_func) + followers = yield [gen.Task(Follower.Query, self._client, f_id, viewpoint.viewpoint_id, None) + for f_id in follower_ids] + + # Gather activities. + # _RepairBadCoverPhoto() depends on the order of activities produced by this query. If it + # changes, _RepairBadCoverPhoto() needs to be modified to compensate. + query_func = partial(Activity.RangeQuery, self._client, viewpoint.viewpoint_id, + range_desc=None, col_names=None) + activities = yield gen.Task(self._CacheQuery, query_func) + + # Gather episodes and photos. + query_func = partial(Viewpoint.QueryEpisodes, self._client, viewpoint.viewpoint_id) + episodes = yield gen.Task(self._CacheQuery, query_func) + ep_photos_list = [] + for episode in episodes: + query_func = partial(Post.RangeQuery, self._client, episode.episode_id, + range_desc=None, col_names=None) + posts = yield gen.Task(self._CacheQuery, query_func) + photos = yield [gen.Task(Photo.Query, self._client, post.photo_id, col_names=None) for post in posts] + + user_posts = None + if viewpoint.IsDefault(): + # Only query user_post entries if this is a default viewpoint. + user_posts = yield [gen.Task(UserPost.Query, self._client, episode.user_id, + Post.ConstructPostId(post.episode_id, post.photo_id), + None, must_exist=False) for post in posts] + + ep_photos_list.append((episode, photos, posts, user_posts)) + if (len(ep_photos_list) % 100) == 0: + logging.info(' caching photos from %d+ episode records...' % len(ep_photos_list)) + + # Gather comments. + query_func = partial(Comment.RangeQuery, self._client, viewpoint.viewpoint_id, + range_desc=None, col_names=None) + comments = yield gen.Task(self._CacheQuery, query_func) + + # Gather accounting entries for this viewpoint. + query_func = partial(Accounting.RangeQuery, self._client, + '%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint.viewpoint_id), + range_desc=None, col_names=None) + accounting = yield gen.Task(self._CacheQuery, query_func) + + if viewpoint.IsDefault(): + # Default viewpoint has a viewpoint-level OWNED_BY accounting entry matching exactly + # the user-level OWNED_BY entry for the viewpoint owner. Look it up now. + user_accounting = yield gen.Task(Accounting.Query, self._client, + '%s:%d' % (Accounting.USER_SIZE, viewpoint.user_id), + Accounting.OWNED_BY, None, must_exist=False) + if user_accounting is not None: + # Add it to the accounting list. It will be automatically checked in CheckBadViewpointAccounting. + accounting.append(user_accounting) + else: + # Set each follower id in visited_users. We only do this for shared viewpoint as the per-user + # actions do not process default viewpoint accounting. + for f_id in follower_ids: + self._visited_users[f_id] = viewpoint.viewpoint_id + + refreshed_viewpoint = yield gen.Task(Viewpoint.Query, self._client, viewpoint.viewpoint_id, None) + # TODO(marc): check if other fields have changed? + + # Now that assets have been gathered, check to see if viewpoint was modified during the gathering phase. + if _TEST_MODE or refreshed_viewpoint.last_updated <= time.time() - _CHECK_THRESHOLD_TIMESPAN: + # Check for invalid viewpoint metadata. + yield gen.Task(self._CheckInvalidViewpointMetadata, viewpoint, activities) + + # Check for multiple share_new activities. + yield gen.Task(self._CheckMultipleShareNew, viewpoint, activities) + + # Check for missing activities. + yield gen.Task(self._CheckMissingActivities, viewpoint, activities, followers, ep_photos_list, comments) + + # Check for missing posts referenced by activities. + yield gen.Task(self._CheckMissingPosts, viewpoint, activities, ep_photos_list) + + # Check for any missing Followed records. + yield gen.Task(self._CheckMissingFollowed, viewpoint, followers) + + # Check for empty viewpoint. + yield gen.Task(self._CheckEmptyViewpoint, viewpoint, activities, followers, ep_photos_list, comments) + + # Check for missing/bad accounting entries. + yield gen.Task(self._CheckBadViewpointAccounting, viewpoint, ep_photos_list, accounting) + + if not viewpoint.IsDefault(): + # Check for valid cover_photo. + yield gen.Task(self._CheckBadCoverPhoto, viewpoint, ep_photos_list, activities) + else: + logging.info('Aborting viewpoint "%s" check because it was modified while gathering its assets...', + viewpoint.viewpoint_id) + else: + logging.info('Skipping viewpoint "%s" because it was modified in the last hour...', viewpoint.viewpoint_id) + + # Check users in visited_users if it has grown big enough. + self._current_viewpoint = '' + yield gen.Task(self._MaybeCheckVisitedUsers) + callback() + + @gen.engine + def _CheckMultipleShareNew(self, viewpoint, activities, callback): + """Checks that a viewpoint has at most one share_new activity.""" + # Gather any duplicate share_new activities. + earliest_activity = None + dup_activities = [] + for activity in activities: + if activity.name == 'share_new': + if earliest_activity is None: + earliest_activity = activity + elif activity.timestamp < earliest_activity.timestamp and activity.user_id == viewpoint.user_id: + dup_activities.append(earliest_activity) + earliest_activity = activity + else: + dup_activities.append(activity) + + # Report corruption for each duplicate. + if len(dup_activities) > 0: + for activity in dup_activities: + # Only support empty or matching or redundant follower_ids. + follower_ids = json.loads(activity.json)['follower_ids'] + assert len(follower_ids) == 0 or follower_ids == json.loads(earliest_activity.json)['follower_ids'] or \ + (len(follower_ids) == 1 and follower_ids[0] == viewpoint.user_id), \ + (activity, earliest_activity) + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'multiple share_new activities', (earliest_activity.activity_id, activity.activity_id), + partial(self._RepairMultipleShareNew, activity)) + callback() + + @gen.engine + def _RepairMultipleShareNew(self, activity, callback): + """Replaces extraneous share_new activities with corresponding + share_existing activity. + """ + assert activity.name == 'share_new', activity + act_args = json.loads(activity.json) + del act_args['follower_ids'] + + # Update share_new => share_existing. + update_activity = Activity.CreateFromKeywords(viewpoint_id=activity.viewpoint_id, + activity_id=activity.activity_id, + name='share_existing', + json=json.dumps(act_args)) + yield gen.Task(update_activity.Update, self._client) + logging.warning(' updated activity: %s', activity.activity_id) + + callback() + + @gen.engine + def _CheckMissingActivities(self, viewpoint, activities, followers, ep_photos_list, comments, callback): + """Checks that every follower, episode, photo, and comment is + referenced at least once by an activity. + + TODO: Consider checking for missing unshare activities. + """ + # Index the activity contents. + index = set() + + # Viewpoint creator is automatically a follower. + index.add(viewpoint.user_id) + + has_share_new = False + for activity in activities: + if activity.name == 'share_new': + has_share_new = True + + invalidate = json.loads(activity.json) + if activity.name in ['add_followers', 'share_new']: + [index.add(f_id) for f_id in invalidate['follower_ids']] + if activity.name == 'merge_accounts': + [index.add(invalidate['target_user_id'])] + if activity.name == 'post_comment': + index.add(invalidate['comment_id']) + if activity.name in ['share_existing', 'share_new', 'save_photos']: + for item in invalidate['episodes']: + index.add(item['episode_id']) + [index.add(ph_id) for ph_id in item['photo_ids']] + if activity.name in ['upload_episode']: + index.add(invalidate['episode_id']) + [index.add(ph_id) for ph_id in invalidate['photo_ids']] + + # Iterate through each of the viewpoint assets and make sure they're "covered" by an activity. + missing_follower_ids = [f.user_id for f in followers if f.user_id not in index] + + # Only check viewpoints with content. + if len(ep_photos_list) > 0 or len(comments) > 0: + for episode, photos, _, _ in ep_photos_list: + if episode.episode_id not in index: + if viewpoint.IsDefault(): + # Default viewpoints have upload_episode and save_photos activities. + if episode.parent_ep_id is None: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing upload_episode activity', episode.episode_id, + partial(self._RepairMissingActivities, episode.user_id, viewpoint, + 'upload_episode', (episode, photos))) + else: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing save_photos activity', episode.episode_id, + partial(self._RepairMissingActivities, episode.user_id, viewpoint, + 'save_photos', (episode, photos))) + elif not has_share_new: + # First share activity is share_new. + has_share_new = True + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing share_new activity', episode.episode_id, + partial(self._RepairMissingActivities, episode.user_id, viewpoint, + 'share_new', (episode, photos, missing_follower_ids))) + missing_follower_ids = None + else: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing share_existing activity', episode.episode_id, + partial(self._RepairMissingActivities, episode.user_id, viewpoint, + 'share_existing', (episode, photos, None))) + + if missing_follower_ids: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing add_followers activity', missing_follower_ids, + partial(self._RepairMissingActivities, viewpoint.user_id, viewpoint, + 'add_followers', missing_follower_ids)) + + for comment in comments: + if comment.comment_id not in index: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'missing post_comment activity', comment.comment_id, + partial(self._RepairMissingActivities, comment.user_id, viewpoint, + 'post_comment', comment.comment_id)) + + callback() + + @gen.engine + def _RepairMissingActivities(self, user_id, viewpoint, name, act_args, callback): + """Adds a missing activity to the specified viewpoint.""" + timestamp = util.GetCurrentTimestamp() + unique_id = yield gen.Task(Device.AllocateSystemObjectId, self._client) + activity_id = Activity.ConstructActivityId(timestamp, Device.SYSTEM, unique_id) + + if name == 'add_followers': + yield gen.Task(Activity.CreateAddFollowers, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, follower_ids=act_args) + elif name == 'upload_episode': + episode, photos = act_args + ep_dict = {'episode_id': episode.episode_id} + ph_dicts = [{'photo_id': photo.photo_id} for photo in photos] + yield gen.Task(Activity.CreateUploadEpisode, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, ep_dict=ep_dict, ph_dicts=ph_dicts) + elif name in ['save_photos']: + episode, photos = act_args + ep_dict = {'new_episode_id': episode.episode_id, + 'photo_ids': [ph.photo_id for ph in photos]} + yield gen.Task(Activity.CreateSavePhotos, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, ep_dicts=[ep_dict]) + elif name in ['share_new', 'share_existing']: + episode, photos, follower_ids = act_args + ep_dict = {'new_episode_id': episode.episode_id, + 'photo_ids': [ph.photo_id for ph in photos]} + if name == 'share_new': + yield gen.Task(Activity.CreateShareNew, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, ep_dicts=[ep_dict], + follower_ids=follower_ids) + else: + yield gen.Task(Activity.CreateShareExisting, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, ep_dicts=[ep_dict]) + elif name == 'post_comment': + yield gen.Task(Activity.CreatePostComment, self._client, user_id, viewpoint.viewpoint_id, + activity_id, timestamp, update_seq=0, cm_dict={'comment_id': act_args}) + + logging.warning(' added activity: %s', (viewpoint.viewpoint_id, name)) + callback() + + @gen.coroutine + def _CheckMissingPosts(self, viewpoint, activities, ep_photos_list): + """Check activities for missing posts.""" + # Build a lookup of photos for this viewpoint, grouped by episode, that we know exist. + existing_posts = defaultdict(set) + for _, photos, posts, _ in ep_photos_list: + for post in posts: + existing_posts[post.episode_id].add(post.photo_id) + + for activity in activities: + if activity.name in ['share_new', 'share_existing']: + for ep_dict in json.loads(activity.json)['episodes']: + if len(ep_dict['photo_ids']) > 0: + if ep_dict['episode_id'] not in existing_posts: + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'no posts found for episode referenced by activity', + (ep_dict['episode_id'], activity.activity_id), + partial(self._RepairMissingPosts, activities, existing_posts)) + raise gen.Return() + for photo_id in ep_dict['photo_ids']: + if photo_id not in existing_posts[ep_dict['episode_id']]: + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'missing post referenced by activity', + (ep_dict['episode_id'], photo_id, activity.activity_id), + partial(self._RepairMissingPosts, activities, existing_posts)) + raise gen.Return() + + @gen.coroutine + def _RepairMissingPosts(self, activities, existing_posts): + """Remove references to posts in activities if the posts don't exist.""" + @gen.coroutine + def _RebuildActivity(activity, act_args): + rebuilt_episodes = [] + for ep_dict in act_args['episodes']: + if ep_dict['episode_id'] in existing_posts: + new_ep_dict = {'episode_id': ep_dict['episode_id']} + for photo_id in list(ep_dict['photo_ids']): + if photo_id in existing_posts[ep_dict['episode_id']]: + new_ep_dict.setdefault('photo_ids', []).append(photo_id) + rebuilt_episodes.append(new_ep_dict) + else: + # Rebuilding the activity is problematic without any posts in an episode. Creating an activity with an + # empty episode will lead to other dbchk errors. Currently, there are no known cases of this, so we'll + # just log it and skip processing of this activity for now. If we ever hit a + # case of this, we can address it then. + logging.warning(' unable to rebuild activity: %s; missing all posts for episode: %s', + activity.activity_id, + ep_dict['episode_id']) + return + act_args['episodes'] = rebuilt_episodes + update_activity = Activity.CreateFromKeywords(viewpoint_id=activity.viewpoint_id, + activity_id=activity.activity_id, + name=activity.name, + json=json.dumps(act_args)) + yield gen.Task(update_activity.Update, self._client) + logging.warning(' updated activity: %s', activity.activity_id) + + def _IsEpisodeMissingAnyPosts(ep_dict): + for photo_id in ep_dict['photo_ids']: + if photo_id not in existing_posts[ep_dict['episode_id']]: + return True + return False + + for activity in activities: + if activity.name in ['share_new', 'share_existing']: + act_args = json.loads(activity.json) + for ep_dict in act_args['episodes']: + if ep_dict['episode_id'] not in existing_posts or _IsEpisodeMissingAnyPosts(ep_dict): + yield _RebuildActivity(activity, act_args) + # Move on to next activity. + break + + @gen.engine + def _CheckInvalidViewpointMetadata(self, viewpoint, activities, callback): + """Checks correctness of viewpoint metadata: + 1. last_updated must be defined + 2. timestamp must be defined + """ + if viewpoint.last_updated is None: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, 'invalid viewpoint metadata', 'last_updated', + partial(self._RepairInvalidViewpointMetadata, viewpoint, activities)) + + if viewpoint.timestamp is None: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, 'invalid viewpoint metadata', 'timestamp', + partial(self._RepairInvalidViewpointMetadata, viewpoint, activities)) + + callback() + + @gen.engine + def _RepairInvalidViewpointMetadata(self, viewpoint, activities, callback): + """Repairs invalid viewpoint metadata: + 1. sets last_updated if it was not defined + 2. sets timestamp if it was not defined + """ + timestamp = min(a.timestamp for a in activities) if activities else 0 + if viewpoint.last_updated is None: + viewpoint.last_updated = timestamp + yield gen.Task(viewpoint.Update, self._client) + logging.warning(' set last_updated to %d: %s', viewpoint.last_updated, viewpoint.viewpoint_id) + + if viewpoint.timestamp is None: + viewpoint.timestamp = timestamp + yield gen.Task(viewpoint.Update, self._client) + logging.warning(' set timestamp to %d: %s', viewpoint.timestamp, viewpoint.viewpoint_id) + + callback() + + @gen.coroutine + def _CheckBadCoverPhoto(self, viewpoint, ep_photos_list, activities): + """Ensure that the cover photo is valid for this viewpoint.""" + # Qualified means that a post is not removed (which implies that it's not unshared, either). + has_qualified_posts = False + if viewpoint.IsCoverPhotoSet(): + cp_episode_id = viewpoint.cover_photo.get('episode_id') + cp_photo_id = viewpoint.cover_photo.get('photo_id') + if cp_episode_id is None or cp_photo_id is None: + # If Viewpoint.cover_photo is not None, these should be present and not None. + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'viewpoint cover_photo is not None, but does not have proper keys', + viewpoint.cover_photo, + partial(self._RepairBadCoverPhoto, viewpoint, ep_photos_list, activities)) + raise gen.Return() + else: + cp_episode_id = None + cp_photo_id = None + + for _, _, posts, _ in ep_photos_list: + for post in posts: + if not post.IsRemoved(): + # This post is qualified to be a cover_photo. + has_qualified_posts = True + if post.photo_id == cp_photo_id and post.episode_id == cp_episode_id: + # Found qualified matching post. Terminate check of this viewpoint. + raise gen.Return() + elif post.photo_id == cp_photo_id and post.episode_id == cp_episode_id: + # Found a match, but of a non-qualifying post. + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'viewpoint cover_photo is not qualified to be a cover_photo', + post, + partial(self._RepairBadCoverPhoto, viewpoint, ep_photos_list, activities)) + raise gen.Return() + if viewpoint.IsCoverPhotoSet(): + # The cover_photo that is set doesn't refer to any photos within the viewpoint, qualified or not. + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'viewpoint cover_photo does not match any photo in viewpoint', + viewpoint.cover_photo, + partial(self._RepairBadCoverPhoto, viewpoint, ep_photos_list, activities)) + elif has_qualified_posts: + # No cover photo set, but there are photos available to the cover photo. + yield gen.Task(self._ReportCorruption, + viewpoint.viewpoint_id, + 'viewpoint cover_photo is set to None and there are qualified photos available', + viewpoint, + partial(self._RepairBadCoverPhoto, viewpoint, ep_photos_list, activities)) + + @gen.coroutine + def _RepairBadCoverPhoto(self, viewpoint, ep_photos_list, activities): + """Select a new cover photo for this viewpoint. Or clear it if none are available.""" + # Build dictionary of shared posts for looking up posts by PostId: + shared_posts = {Post.ConstructPostId(post.episode_id, post.photo_id) : post + for _, _, posts, _ in ep_photos_list + for post in posts if not post.IsRemoved()} + + # This assumes that the activities list was generated from a forward scan query so we reverse it here. + cover_photo = yield gen.Task(viewpoint.SelectCoverPhoto, + self._client, + set(), + activities_list=reversed(activities), + available_posts_dict=shared_posts) + viewpoint.cover_photo = cover_photo + yield gen.Task(viewpoint.Update, self._client) + logging.warning(' updating cover_photo: %s, %s', viewpoint.viewpoint_id, cover_photo) + + @gen.engine + def _CheckEmptyViewpoint(self, viewpoint, activities, followers, ep_photos_list, comments, callback): + """Checks for empty viewpoint.""" + if not viewpoint.IsDefault() and not activities and not ep_photos_list and not comments: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, 'empty viewpoint', None, + partial(self._RepairEmptyViewpoint, viewpoint, followers)) + + callback() + + @gen.engine + def _RepairEmptyViewpoint(self, viewpoint, followers, callback): + """Deletes a corrupted viewpoint.""" + for follower in followers: + sort_key = Followed.CreateSortKey(viewpoint.viewpoint_id, viewpoint.last_updated or 0) + followed = yield gen.Task(Followed.Query, self._client, follower.user_id, sort_key, None, must_exist=False) + + # Delete the followed object if it exists. + if followed is not None: + yield gen.Task(followed.Delete, self._client) + logging.warning(' deleted followed: %s', str((followed.user_id, followed.sort_key))) + + # Delete the follower object if it exists. + if follower is not None: + yield gen.Task(follower.Delete, self._client) + logging.warning(' deleted follower: %s', str((follower.user_id, viewpoint.viewpoint_id))) + + # Delete the viewpoint object. + yield gen.Task(viewpoint.Delete, self._client) + logging.warning(' deleted viewpoint: %s', viewpoint.viewpoint_id) + + callback() + + @gen.engine + def _CheckMissingFollowed(self, viewpoint, followers, callback): + """Checks for missing followed records.""" + for follower in followers: + sort_key = Followed.CreateSortKey(viewpoint.viewpoint_id, viewpoint.last_updated or 0) + followed = yield gen.Task(Followed.Query, + self._client, + follower.user_id, + sort_key, + None, + must_exist=False) + if followed is None: + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, 'missing followed', + (follower.user_id, sort_key), + partial(self._RepairMissingFollowed, viewpoint, follower, sort_key)) + + callback() + + @gen.engine + def _RepairMissingFollowed(self, viewpoint, follower, sort_key, callback): + """Adds a Followed object for the specified user and viewpoint.""" + # Add the new Followed object. + yield gen.Task(Followed.UpdateDateUpdated, + self._client, + follower.user_id, + viewpoint.viewpoint_id, + None, + viewpoint.last_updated or 0) + logging.warning(' added followed: %s' % str((follower.user_id, sort_key))) + + # Invalidate the viewpoint so that user's device will reload it. + yield self._CreateNotification(follower.user_id, + 'dbchk add_followed', + NotificationManager._CreateViewpointInvalidation(viewpoint.viewpoint_id)) + + callback() + + @gen.engine + def _CheckBadViewpointAccounting(self, viewpoint, ep_photos_list, accounting_list, callback): + """Compute all accounting entries from ep_photos_list and verify that they match the entries in 'accounting'.""" + act_dict = {} + + def _IncrementAccountingWith(hash_key, sort_key, increment_from): + # Do not create entries if stats are zero. + if increment_from.num_photos == 0: + return + key = (hash_key, sort_key) + act_dict.setdefault(key, Accounting(hash_key, sort_key)).IncrementStatsFrom(increment_from) + + for episode, photos, posts, user_posts in ep_photos_list: + act = Accounting() + act.IncrementFromPhotos([photo for photo, post in zip(photos, posts) if not post.IsRemoved()]) + if viewpoint.IsDefault(): + # Default viewpoint: compute viewpoint-level owned-by: and user-level owned-by. + _IncrementAccountingWith('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint.viewpoint_id), + '%s:%d' % (Accounting.OWNED_BY, episode.user_id), + act) + _IncrementAccountingWith('%s:%d' % (Accounting.USER_SIZE, viewpoint.user_id), + Accounting.OWNED_BY, act) + else: + # Shared viewpoint: compute viewpoint-level shared-by: and visible-to. User-level + # accounting for those categories sums multiple viewpoint-level entries, so cannot be computed here. + _IncrementAccountingWith('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint.viewpoint_id), + '%s:%d' % (Accounting.SHARED_BY, episode.user_id), + act) + _IncrementAccountingWith('%s:%s' % (Accounting.VIEWPOINT_SIZE, viewpoint.viewpoint_id), + Accounting.VISIBLE_TO, + act) + + # Iterate over all existing accounting entries. + for act in accounting_list: + key = (act.hash_key, act.sort_key) + built_act = act_dict.get(key, None) + if built_act is None: + # Entries can drop to 0 when photos get removed or unshared. However, we need to keep + # such entries around to properly handle operation replays. + built_act = Accounting(hash_key=act.hash_key, sort_key=act.sort_key) + + if not built_act.StatsEqual(act): + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, + 'wrong accounting', '%s != %s' % (act, built_act), + partial(self._RepairBadViewpointAccounting, 'update', built_act)) + + # Remove entry from the built accounting entries. + act_dict.pop(key, None) + + # Iterate over the remaining built accounting entries. + for built_act in act_dict.values(): + yield gen.Task(self._ReportCorruption, viewpoint.viewpoint_id, 'missing accounting', built_act, + partial(self._RepairBadViewpointAccounting, 'add', built_act)) + + callback() + + @gen.engine + def _RepairBadViewpointAccounting(self, action, accounting, callback): + if action == 'update': + # Update wrong accounting entry. + yield gen.Task(accounting.Update, self._client) + logging.warning(' updated accounting entry: (%s, %s)', accounting.hash_key, accounting.sort_key) + elif action == 'add': + # Add missing accounting entry. + yield gen.Task(accounting.Update, self._client) + logging.warning(' added accounting entry: (%s, %s)', accounting.hash_key, accounting.sort_key) + + callback() + + @gen.engine + def _MaybeCheckVisitedUsers(self, callback): + """If visited_users has grown too large, trigger CheckVisitedUsers, otherwise, simply return. + This is called at the end of CheckViewpoint. + """ + if len(self._visited_users) >= DatabaseChecker._MAX_VISITED_USERS: + yield gen.Task(self._CheckVisitedUsers) + callback() + + @gen.engine + def _CheckVisitedUsers(self, callback): + """Check each user in the visited_users set and clear it. Called after processing all viewpoint.""" + logging.info('Processing %d visited users' % len(self._visited_users)) + yield [gen.Task(self._CheckUser, user_id, vp_id) for user_id, vp_id in self._visited_users.iteritems()] + self._visited_users.clear() + + callback() + + @gen.engine + def _CheckUser(self, user_id, viewpoint_id, callback): + """Check a single user. viewpoint_id is the visited viewpoint that last encounter this user.""" + logging.info('Processing user %d' % user_id) + self._current_user = user_id + + # Fetch list of followed viewpoints. + query_func = partial(Follower.RangeQuery, self._client, user_id, range_desc=None, col_names=None) + followed_vps = yield gen.Task(self._CacheQuery, query_func) + + accounting_vt = Accounting.CreateUserVisibleTo(user_id) + accounting_sb = Accounting.CreateUserSharedBy(user_id) + + # Desired sort key in vp accounting. This is used to filter out SHARED_BY other users and OWNED_BY. + vp_sb_sort_key = '%s:%d' % (Accounting.SHARED_BY, user_id) + + for f in followed_vps: + # Only consider followers that are not REMOVED. We don't count the visible_to and shared_by + # contributions from the viewpoint of a REMOVED follower. + if not f.IsRemoved(): + # Fetch list of viewpoint-level accounting entries. This will include OWNED_BY in the default users's + # default viewpoint as well as SHARED_BY for other users. This is probably better than doing two separate + # queries for VISIBLE_TO and this user's SHARED_BY. + query_func = partial(Accounting.RangeQuery, + self._client, + '%s:%s' % (Accounting.VIEWPOINT_SIZE, f.viewpoint_id), + range_desc=None, + col_names=None) + vp_accounting = yield gen.Task(self._CacheQuery, query_func) + for act in vp_accounting: + if act.sort_key == vp_sb_sort_key: + accounting_sb.IncrementStatsFrom(act) + elif act.sort_key == Accounting.VISIBLE_TO: + accounting_vt.IncrementStatsFrom(act) + + # Check existence of activities referenced by notifications. + query_func = partial(Notification.RangeQuery, self._client, user_id, range_desc=None, col_names=None) + user_notifications = yield gen.Task(self._CacheQuery, query_func) + + for n in user_notifications: + if not n.viewpoint_id or not n.activity_id: + continue + + activity = yield gen.Task(Activity.Query, self._client, n.viewpoint_id, n.activity_id, None, must_exist=False) + if activity is None: + # TODO: not sure how to auto-repair this. + yield gen.Task(self._ReportCorruption, n.viewpoint_id, + 'missing activity', 'user notification %s:%s references missing activity %s:%s' % + (user_id, n.notification_id, n.viewpoint_id, n.activity_id), + None) + + # Now fetch the user's accounting entries. + user_sb = yield gen.Task(Accounting.Query, self._client, '%s:%d' % (Accounting.USER_SIZE, user_id), + Accounting.SHARED_BY, None, must_exist=False) + user_vt = yield gen.Task(Accounting.Query, self._client, '%s:%d' % (Accounting.USER_SIZE, user_id), + Accounting.VISIBLE_TO, None, must_exist=False) + + # Check users's accounting entries against aggregated viewpoint entries. + # If the built-up accounting is zero, we do not create missing entries as this both complicates + # tests and increases the chances of conflict. + if user_sb is None: + if accounting_sb.num_photos != 0: + yield gen.Task(self._ReportCorruption, viewpoint_id, 'missing user accounting', accounting_sb, + partial(self._RepairBadViewpointAccounting, 'add', accounting_sb)) + elif not user_sb.StatsEqual(accounting_sb): + yield gen.Task(self._ReportCorruption, viewpoint_id, + 'wrong user accounting', '%s != %s' % (user_sb, accounting_sb), + partial(self._RepairBadViewpointAccounting, 'update', accounting_sb)) + + if user_vt is None: + if accounting_sb.num_photos != 0: + yield gen.Task(self._ReportCorruption, viewpoint_id, 'missing user accounting', accounting_vt, + partial(self._RepairBadViewpointAccounting, 'add', accounting_vt)) + elif not user_vt.StatsEqual(accounting_vt): + yield gen.Task(self._ReportCorruption, viewpoint_id, + 'wrong user accounting', '%s != %s' % (user_vt, accounting_vt), + partial(self._RepairBadViewpointAccounting, 'update', accounting_vt)) + + self._current_user = '' + callback() + + @gen.engine + def _CacheQuery(self, query_func, callback): + """Repeatedly invokes the specified query function that takes an + "excl_start_key" and a "limit" argument for paging. The query + function must return an array of result items, or a tuple of + (items, last_key). Combines the result items from multiple calls + into a single array of results, and invokes the callback with it. + """ + _LIMIT = 100 + excl_start_key = None + all_results = [] + while True: + results = yield gen.Task(query_func, limit=_LIMIT, excl_start_key=excl_start_key) + + # Some query funcs return the items directly, some return a tuple of (items, last_key). + if isinstance(results, tuple): + results, excl_start_key = results + elif len(results) > 0: + excl_start_key = results[-1].GetKey() + + all_results.extend(results) + if len(results) < _LIMIT: + break + + if (len(all_results) % 1000) == 0: + logging.info(' caching %d+ %s records...' % (len(all_results), type(results[0]).__name__.lower())) + + callback(all_results) + + @gen.engine + def _ReportCorruption(self, viewpoint_id, name, args, repair_func, callback): + """Logs the corruption and sends occasional emails summarizing any + corruption that is found. + """ + args = '' if args is None else ' (%s)' % str(args) + logging.error('Found database corruption in viewpoint %s: %s%s' % (viewpoint_id, name, args)) + logging.error(' python dbchk.py --devbox --repair=True --viewpoints=%s' % viewpoint_id) + + # Accumulate repairs for later email. + self._corruptions.setdefault(viewpoint_id, {}).setdefault(name, 0) + self._corruptions[viewpoint_id][name] += 1 + + # If it has been a sufficient interval of time, send notification email. + if util.GetCurrentTimestamp() > self._last_email + DatabaseChecker._MAX_EMAIL_INTERVAL: + yield gen.Task(self._SendEmail) + + if self._repair: + if repair_func is not None: + logging.warning('Repairing corruption: %s' % name) + yield gen.Task(repair_func) + logging.info('') + else: + logging.warning('No repair function for corruption: %s' % name) + + callback() + + @gen.engine + def _SendEmail(self, callback): + # If no corruption, don't create email. + if len(self._corruptions) == 0: + callback() + return + + email_text = '' + for i, (viewpoint_id, vp_repair_dict) in enumerate(self._corruptions.iteritems()): + email_text += ' ---- viewpoint %s ----\n' % viewpoint_id + for name, count in vp_repair_dict.iteritems(): + email_text += ' %s (%d instance%s)\n' % (name, count, util.Pluralize(count)) + email_text += '\n' + + if i > 50: + email_text += ' ...too many viewpoints to list\n\n' + + email_text += 'python dbchk.py --devbox --repair=True --viewpoints=%s' % \ + ','.join(self._corruptions.keys()) + + args = { + 'from': 'dbchk@emailscrubbed.com', + 'fromname': 'DB Checker', + 'to': options.options.email, + 'subject': 'Database corruption', + 'text': DatabaseChecker._EMAIL_TEXT % email_text + } + + # In test mode, save email args but don't send email. + # Don't send email if problems will be repaired. + if _TEST_MODE: + self._email_args = args + elif options.options.email and not self._repair: + yield gen.Task(EmailManager.Instance().SendEmail, description=args['subject'], **args) + + self._corruptions = {} + self._last_email = util.GetCurrentTimestamp() + callback() + + @gen.coroutine + def _CreateNotification(self, user_id, name, invalidate): + """Create notification in order to notify user's devices that + content needs to be re-loaded. + """ + # Create dummy operation. + op_id = Operation.ConstructOperationId(Operation.ANONYMOUS_DEVICE_ID, 0) + op = Operation(Operation.ANONYMOUS_USER_ID, op_id) + op.device_id = Operation.ANONYMOUS_DEVICE_ID + op.timestamp = util.GetCurrentTimestamp() + + yield Notification.CreateForUser(self._client, + op, + user_id, + name, + invalidate=invalidate) + + @gen.engine + def _ThrottledScan(self, scan_cls, visitor, callback, col_names=None, scan_filter=None, + consistent_read=False, max_read_units=None): + """Scan over the "scan_cls" table, processing at most "max_read_units" + items per second. Invoke the "visitor" function for each item in the + table. + """ + _SCAN_LIMIT = 50 + + assert max_read_units is None or max_read_units >= 1.0, max_read_units + start_key = None + num_items = 0.0 + start_time = time.time() + + while True: + items, start_key = yield gen.Task(scan_cls.Scan, self._client, None, limit=_SCAN_LIMIT, + excl_start_key=start_key, scan_filter=scan_filter) + + for item in items: + # Check to see if max_read_units have been exceeded and therefore a delay is needed. + if max_read_units is not None: + now = time.time() + elapsed_time = now - start_time + if elapsed_time == 0.0 or (num_items / elapsed_time) > max_read_units: + # Wait 1 second before proceeding to the next item. + yield gen.Task(IOLoop.current().add_timeout, now + 1) + + # Make the visit for this item. + yield gen.Task(visitor, item) + + # Track number of items that have been processed. + num_items += 1.0 + if (num_items % 10) == 0: + logging.info('Scanned %d %ss...' % (num_items, scan_cls.__name__.lower())) + + # Stop or continue the scan. + if start_key is None: + callback() + return + + +def ThrottleUsage(): + """Ensures that only a portion of total read/write capacity is consumed + by this checker. + """ + for table in vf_schema.SCHEMA.GetTables(): + table.read_units = max(1, table.read_units // 4) + table.write_units = max(1, table.write_units // 4) + +@gen.engine +def RunOnce(client, job, callback): + """Perform a single dbchk run based on the options and previous runs. + We catch all exceptions within the database checker and register the run as a failure. + We must be called with the job lock held or not require locking. + """ + assert not options.options.require_lock or job.HasLock() == True + checker = DatabaseChecker(client, repair=options.options.repair) + + last_scan = None + if options.options.smart_scan: + # Search for successful full-scan run in the last week. + last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.full_scan', with_payload_value=True) + + if last_run is None: + logging.info('No successful run found in the last week; performing full scan.') + else: + # Make sure enough time has passed since the last run. + last_run_start = last_run['start_time'] + if util.HoursSince(last_run_start) < options.options.hours_between_runs: + logging.info('Last successful run started at %s, less than %d hours ago; skipping.' % + (time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs)) + callback() + return + + # Set scan_start to start of previous run - 1h (dbchk does not scan viewpoints updated in the last hour). + last_scan = last_run_start - _CHECK_THRESHOLD_TIMESPAN + # We intentionally log local times to avoid confusion. + logging.info('Last successful DBCHK run was at %s, scanning viewpoints updated after %s' % + (time.asctime(time.localtime(last_run_start)), time.asctime(time.localtime(last_scan)))) + + job.Start() + try: + if options.options.viewpoints: + yield gen.Task(checker.CheckViewpointList, options.options.viewpoints) + else: + yield gen.Task(checker.CheckAllViewpoints, last_scan=last_scan) + except: + # Failure: log run summary with trace. + typ, val, tb = sys.exc_info() + msg = 'Error while visiting viewpoint=%s or user=%s\n' % (checker._current_viewpoint, checker._current_user) + msg += ''.join(traceback.format_exception(typ, val, tb)) + logging.info('Registering failed run with message: %s' % msg) + yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) + else: + # Successful: write run summary. + stats = DotDict() + stats['full_scan'] = len(options.options.viewpoints) == 0 + stats['dbchk.visited_viewpoints'] = checker._num_visited_viewpoints + logging.info('Registering successful run with stats: %r' % stats) + yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) + + callback() + +@gen.engine +def Dispatch(client, callback): + """Dispatches according to command-line options.""" + job = Job(client, 'dbchk') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + try: + yield gen.Task(RunOnce, client, job) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + +@gen.engine +def SetupAndDispatch(callback): + """Sets the environment and dispatches according to command-line options.""" + EmailManager.SetInstance(SendGridEmailManager()) + + # Try not to disturb production usage while checking and repairing the database. + ThrottleUsage() + + client = db_client.DBClient.Instance() + + # Dispatch command-line options. + yield gen.Task(Dispatch, client) + callback() + +if __name__ == '__main__': + sys.exit(main.InitAndRun(SetupAndDispatch)) diff --git a/backend/db/tools/email_marketing.py b/backend/db/tools/email_marketing.py new file mode 100644 index 0000000..7d85fc5 --- /dev/null +++ b/backend/db/tools/email_marketing.py @@ -0,0 +1,169 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import sys + +from functools import partial +from tornado import escape, gen +from tornado.options import options +from tornado.template import Template +from urllib import urlencode +from viewfinder.backend.base import main, util +from viewfinder.backend.db.db_client import DBClient, DBKey +from viewfinder.backend.db.settings import AccountSettings +from viewfinder.backend.db.user import User +from viewfinder.backend.services.email_mgr import EmailManager, SendGridEmailManager +from viewfinder.backend.services.sms_mgr import SMSManager, TwilioSMSManager + +options.define('email_template', default=None, type=str, + help='name of the .email template file to use') +options.define('email_subject', default='New Viewfinder Features', type=str, + help='subject to relay with email message') +options.define('sms_template', default=None, type=str, + help='name of the .sms template file to use') +options.define('min_user_id', default=11, type=int, + help='only users with ids >= this id will be sent email/SMS (-1 for no min)') +options.define('max_user_id', default=11, type=int, + help='only users with ids <= this id will be sent email/SMS (-1 for no max)') +options.define('honor_allow_marketing', default=True, type=bool, + help='do not send the email/SMS if the user has turned off marketing emails') +options.define('test_mode', default=True, type=bool, + help='do not send the email/SMS; print the first email/SMS that would have been sent') + +_is_first_email = True +_is_first_sms = True + + +@gen.coroutine +def GetRegisteredUsers(client, last_user_id): + """Get next batch of users that are registered, and that have a primary email or phone.""" + if options.min_user_id == options.max_user_id and options.min_user_id != -1: + # Shortcut for single user. + if last_user_id is None: + result = [(yield gen.Task(User.Query, client, options.min_user_id, None))] + else: + result = [] + else: + start_key = DBKey(last_user_id, None) if last_user_id is not None else None + result = (yield gen.Task(User.Scan, client, None, excl_start_key=start_key))[0] + + users = [user for user in result if user.IsRegistered() and not user.IsTerminated() and (user.email or user.phone)] + raise gen.Return(users) + + +@gen.coroutine +def SendEmailToUser(template, user): + assert user.email is not None, user + + unsubscribe_cookie = User.CreateUnsubscribeCookie(user.user_id, AccountSettings.MARKETING) + unsubscribe_url = 'https://%s/unsubscribe?%s' % (options.domain, + urlencode(dict(cookie=unsubscribe_cookie))) + + # Create arguments for the email template. + fmt_args = {'first_name': user.given_name, + 'unsubscribe_url': unsubscribe_url} + + # Create arguments for the email. + args = {'from': EmailManager.Instance().GetInfoAddress(), + 'fromname': 'Viewfinder', + 'to': user.email, + 'subject': options.email_subject} + util.SetIfNotNone(args, 'toname', user.name) + + args['html'] = template.generate(is_html=True, **fmt_args) + args['text'] = template.generate(is_html=False, **fmt_args) + + print 'Sending marketing email to %s (%s) (#%d)' % (user.email, user.name, user.user_id) + + if options.test_mode: + global _is_first_email + if _is_first_email: + print args['html'] + _is_first_email = False + else: + # Remove extra whitespace in the HTML (seems to help it avoid Gmail spam filter). + args['html'] = escape.squeeze(args['html']) + + yield gen.Task(EmailManager.Instance().SendEmail, description='marketing email', **args) + + +@gen.coroutine +def SendSMSToUser(template, user): + assert user.phone is not None, user + + # Create arguments for the SMS template. + fmt_args = {'first_name': user.given_name} + + # Create arguments for the SMS. + args = {'number': user.phone, + 'text': template.generate(is_html=False, **fmt_args)} + + print 'Sending marketing SMS to %s (%s) (#%d)' % (user.phone, user.name, user.user_id) + + if options.test_mode: + global _is_first_sms + if _is_first_sms: + print args['text'] + _is_first_sms = False + else: + yield gen.Task(SMSManager.Instance().SendSMS, description='marketing SMS', **args) + + +@gen.engine +def Run(callback): + assert options.email_template, '--email_template must be set' + + EmailManager.SetInstance(SendGridEmailManager()) + SMSManager.SetInstance(TwilioSMSManager()) + + # Load the email template. + f = open(options.email_template, "r") + email_template = Template(f.read()) + f.close() + + # Load the SMS template. + if options.sms_template: + f = open(options.sms_template, "r") + sms_template = Template(f.read()) + f.close() + + sms_warning = False + client = DBClient.Instance() + last_user_id = None + count = 0 + while True: + users = yield GetRegisteredUsers(client, last_user_id) + if not users: + break + + count += len(users) + print 'Scanned %d users...' % count + for user in users: + last_user_id = user.user_id + if options.min_user_id != -1 and user.user_id < options.min_user_id: + continue + + if options.max_user_id != -1 and user.user_id > options.max_user_id: + continue + + # Only send to users which allow marketing communication to them. + if options.honor_allow_marketing: + settings = yield gen.Task(AccountSettings.QueryByUser, client, user.user_id, None) + if not settings.AllowMarketing(): + continue + + if user.email: + yield SendEmailToUser(email_template, user) + elif user.phone: + if not options.sms_template: + if not sms_warning: + print 'WARNING: no SMS template specified and phone-only accounts encountered; skipping...' + sms_warning = True + else: + yield SendSMSToUser(sms_template, user) + + callback() + +if __name__ == '__main__': + sys.exit(main.InitAndRun(Run)) diff --git a/backend/db/tools/raw_dba.py b/backend/db/tools/raw_dba.py new file mode 100644 index 0000000..f807a19 --- /dev/null +++ b/backend/db/tools/raw_dba.py @@ -0,0 +1,201 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utility to list, create, delete & describe tables. + +Usage: + +--op specifies the database operation: + create + REALLY_DELETE_TABLE + list + describe + delete-item + get-item + update-item + delete-range + scan + query + +Hash key and range key (--hash_key, --range_key) are eval'd. This means +that string values should be quoted. An example with a string as hash +key and an integer as range key: + +--hash_key="'Email:spencer.kimball@emailscrubbed.com'" --range_key=5 + +Attributes for update-item are specified with the actual column names (NOT keys!). +This is a comma-separated list of column-name/colum-value pairs. Each column value +will be eval'd in python--this allows sets to be specified, as well as None, etc. + +The quoting from the command line can be tricky. Here's an example of how to get +it to work. The trick is to put double quote around the entire comma-separated +list and then escape any double-quoted attribute values inside: + +--attributes="col_names=\"set([u'photo_id', u'episode_id', u'timestamp', u'placemark', u'_version', u'client_data', u'location', u'content_type', u'aspect_ratio', u'user_id'])\",labels=\"set([u'+owned'])\",_version=0,user_id=7,user_update_id=528,photo_id=\"'pgBXxbGuh-F'\"" + + +python -m viewfinder.backend.db.tools.raw_dba \ + [--tables=table0[,table1[,...]]] [--op=] + +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import pprint +import sys + +from functools import partial +from tornado import options +from viewfinder.backend.base import main, util +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.tools.util import AttrParser + +options.define('tables', default=None, multiple=True, + help='list of tables to upgrade; "ALL" for all tables') +options.define('verify_or_create', default=True, + help='verify or create schema on database') +options.define('op', default='list', + help='DBA operation on tables; one of (create, delete, list, describe)') +options.define('hash_key', default=None, help='primary hash key for item-level ops') +options.define('range_key', default=None, help='primary range key for item-level ops') +options.define('attributes', default=None, help='value attributes pairs (attr0=value0,attr1=value1,...) ' + 'can be quoted strings. Values are eval\'d') +options.define('col_names', default=None, multiple=True, help='column names to print when querying items') + + +def Scan(client, table, callback): + """Scans an entire table. + """ + def _OnScan(retry_cb, count, result): + for item in result.items: + if options.options.col_names: + item = dict([(k, v) for k, v in item.items() if k in options.options.col_names]) + pprint.pprint(item) + if result.last_key: + retry_cb(result.last_key, count + len(result.items)) + else: + return callback('scanned %d items' % (count + len(result.items))) + + def _Scan(last_key=None, count=0): + client.Scan(table.name, partial(_OnScan, _Scan, count), + None, limit=50, excl_start_key=last_key) + + _Scan() + +def QueryRange(client, table, callback): + """Queries the contents of a range identified by --hash_key. + """ + def _OnQuery(retry_cb, count, result): + for item in result.items: + if options.options.col_names: + item = dict([(k, v) for k, v in item.items() if k in options.options.col_names]) + pprint.pprint(item) + if result.last_key: + retry_cb(result.last_key, count + len(result.items)) + else: + return callback('queried %d items' % count) + + def _Query(last_key=None, count=0): + client.Query(table.name, options.options.hash_key, None, partial(_OnQuery, _Query, count), + None, limit=100, excl_start_key=last_key) + + _Query() + +def DeleteRange(client, table, callback): + """Deletes all items in the range identified by --hash_key. + """ + def _OnQuery(retry_cb, count, result): + count += len(result.items) + if not result.last_key: + result_cb = partial(callback, 'deleted %d items' % count) + else: + logging.info('deleting next %d items from %s' % (len(result.items), table.name)) + result_cb = partial(retry_cb, result.last_key, count) + with util.Barrier(result_cb) as b: + for item in result.items: + key = db_client.DBKey(options.options.hash_key, item[table.range_key_col.key]) + client.DeleteItem(table=table.name, key=key, callback=b.Callback()) + + def _Query(last_key=None, count=0): + client.Query(table.name, options.options.hash_key, None, partial(_OnQuery, _Query, count), + None, limit=100, excl_start_key=last_key) + + _Query() + + +def RunOpOnTable(client, table, op, callback): + """Runs the specified op on the table.""" + if options.options.hash_key and options.options.range_key: + key = db_client.DBKey(eval(options.options.hash_key), eval(options.options.range_key)) + elif options.options.hash_key: + key = db_client.DBKey(eval(options.options.hash_key), None) + else: + key = None + + def _OnOp(result): + logging.info('%s: %s' % (table.name, repr(result))) + callback() + + logging.info('executing %s on table %s' % (op, table.name)) + if op == 'create': + client.CreateTable(table=table.name, hash_key_schema=table.hash_key_schema, + range_key_schema=table.range_key_schema, + read_units=table.read_units, write_units=table.write_units, + callback=_OnOp) + elif op == 'describe': + client.DescribeTable(table=table.name, callback=_OnOp) + elif op == 'REALLY_DELETE_TABLE': + client.DeleteTable(table=table.name, callback=_OnOp) + elif op == 'get-item': + client.GetItem(table=table.name, key=key, callback=_OnOp, + attributes=options.options.col_names) + elif op == 'update-item': + parser = AttrParser(table, raw=True) + attrs = parser.Run(options.options.attributes) + client.UpdateItem(table=table.name, key=key, callback=_OnOp, + attributes=attrs, return_values='ALL_NEW') + elif op == 'delete-item': + client.DeleteItem(table=table.name, key=key, callback=_OnOp, + return_values='ALL_OLD') + elif op == 'delete-range': + assert table.range_key_col, 'Table %s is not composite' % table.name + DeleteRange(client, table, _OnOp) + elif op == 'query': + assert table.range_key_col, 'Table %s is not composite' % table.name + QueryRange(client, table, _OnOp) + elif op == 'scan': + Scan(client, table, _OnOp) + else: + raise Exception('unrecognized op: %s; ignoring...' % op) + + +def RunDBA(callback): + """Runs op on each table listed in --tables.""" + logging.warning('WARNING: this tool can modify low-level DynamoDB tables and ' + 'attributes and should be used with caution. For example, ' + 'modifying a photo or adding a label directly will ' + 'not update secondary indexes nor create user updates.') + + def _OnInit(verified_schema): + if options.options.op == 'list': + def _OnList(result): + logging.info(result) + callback() + db_client.DBClient.Instance().ListTables(callback=_OnList) + else: + if options.options.tables == 'ALL': + tables = vf_schema.SCHEMA.GetTables() + else: + tables = [vf_schema.SCHEMA.GetTable(n) for n in options.options.tables] + assert tables, 'no tables were specified' + + with util.Barrier(callback) as b: + for table in tables: + RunOpOnTable(db_client.DBClient.Instance(), table, options.options.op, b.Callback()) + + db_client.InitDB(vf_schema.SCHEMA, callback=_OnInit, + verify_or_create=options.options.verify_or_create) + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(RunDBA, init_db=False)) diff --git a/backend/db/tools/upgrade.py b/backend/db/tools/upgrade.py new file mode 100644 index 0000000..266a735 --- /dev/null +++ b/backend/db/tools/upgrade.py @@ -0,0 +1,132 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Upgrades a table (or tables) in the database by scanning +sequentially through and updating each item. This relies on table +features defined in the Schema for each table. 'Features' are a +collection of rank-ordered tags, each with a corresponding alteration +to the data model. This may involve the addition of a new column, the +deprecation and removal of an existing column, and update to the +indexing algorithm, or some transformation of the data. + +Each item's '_version' column is checked against the feature tags +defined for the table. All features with higher version numbers than +the item's current version are applied to the item in ordinal +succession. The item is then updated in the database. + +Table names for the --tables flag are class names (i.e. no underscores). + +Usage: + +python -m viewfinder.backend.db.tools.upgrade [--tables=table0[,table1[,...]]] + +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import sys + +from tornado import options, gen +from viewfinder.backend.base import main +from viewfinder.backend.db import db_client, db_import, versions, vf_schema +from viewfinder.backend.db.versions import Version +from viewfinder.backend.op.op_manager import OpManager +from viewfinder.backend.op.operation_map import DB_OPERATION_MAP + +options.define('tables', default='', multiple=True, + help='list of tables to upgrade; leave blank for all') +options.define('scan_limit', default=100, + help='maximum number of items per scan') +options.define('upgrade_limit', default=None, + help='maximum number of items to upgrade; do not specify to upgrade all') +options.define('verbose_test_run', default=True, help='set to False to mutate the database; ' + 'otherwise runs in verbose test mode') +options.define('allow_s3_queries', default=None, type=bool, + help='allow S3 queries in upgrades. Involves filling metadata from S3. ' + 'Current use it for versions CREATE_MD5_HASHES and FILL_FILE_SIZES.') +options.define('excl_start_key', default=None, type=str, + help='Start scanning from this key') +options.define('migrator', default=None, type=str, + help='use the given migrator (a name from backend.db.versions) for each table processed') + +@gen.engine +def UpgradeTable(client, table, callback): + """Sequentially scans 'table', updating each scan item to trigger + necessary upgrades. + """ + upgrade_versions = [] + if options.options.migrator: + upgrade_versions.append(getattr(versions, options.options.migrator)) + else: + raise Exception('Upgrade requires the --migrator option.') + if not db_import.GetTableClass(table.name): + raise Exception('Upgrade is not supported on table %s.' % table.name) + + # Get full count of rows in the database for logging progress. + describe = yield gen.Task(client.DescribeTable, table=table.name) + logging.info('%s: (%d items)' % (table.name, describe.count)) + if options.options.excl_start_key: + excl_start_key = db_client.DBKey(options.options.excl_start_key, None) + else: + excl_start_key = None + + # Loop while scanning in batches. Scan will have already updated the + # items if it was needed, so there is no need to call MaybeMigrate + # again. If 'last_key' is None, the scan is complete. + # Otherwise, continue with scan by supplying the last key as the exclusive start key. + count = 0 + scan_params = {'client': client, + 'col_names': None, + 'limit': options.options.scan_limit, + 'excl_start_key': excl_start_key} + while True: + items, last_key = yield gen.Task(db_import.GetTableClass(table.name).Scan, **scan_params) + + # Maybe migrate all items. + yield [gen.Task(Version.MaybeMigrate, client, item, upgrade_versions) + for item in items] + + logging.info('scanned next %d items from table %s' % (len(items), table.name)) + + new_count = count + len(items) + logging.info('processed a total of %d items from table %s' % (new_count, table.name)) + # Log a progress notification every 10% scanned. + if describe.count and (new_count * 10) / describe.count > (count * 10) / describe.count: + logging.info('%s: %d%%%s' % + (table.name, (new_count * 100) / describe.count, + '...' if new_count != describe.count else '')) + if last_key is None: + break + elif options.options.upgrade_limit and new_count >= int(options.options.upgrade_limit): + logging.info('exhausted --upgrade_limit=%s; exiting...' % options.options.upgrade_limit) + break + + # Prepare for next iteration of loop. + scan_params['excl_start_key'] = last_key + count = new_count + + callback() + +@gen.engine +def Upgrade(callback): + """Upgrades each table in 'options.options.tables'.""" + if options.options.verbose_test_run: + logging.info('***** NOTE: upgrade is being run in verbose testing mode; run with ' + '--verbose_test_run=False once changes have been verified') + Version.SetMutateItems(False) + if options.options.allow_s3_queries is not None: + logging.info('Setting allow_s3_queries=%r' % options.options.allow_s3_queries) + Version.SetAllowS3Queries(options.options.allow_s3_queries) + + OpManager.SetInstance(OpManager(op_map=DB_OPERATION_MAP, client=db_client.DBClient.Instance())) + + tables = [vf_schema.SCHEMA.GetTable(n) for n in options.options.tables] + if not tables: + raise Exception('The --tables option has not been specified. ' + + 'Tables to upgrade must be explicitly listed using this option.') + yield [gen.Task(UpgradeTable, db_client.DBClient.Instance(), table) for table in tables] + + callback() + +if __name__ == '__main__': + sys.exit(main.InitAndRun(Upgrade)) diff --git a/backend/db/tools/util.py b/backend/db/tools/util.py new file mode 100644 index 0000000..82b3dbe --- /dev/null +++ b/backend/db/tools/util.py @@ -0,0 +1,63 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utilities module for use from db/tools. + + - AttrParser: parser for attribute key/value pairs +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import logging +import signal + +from picoparse import one_of, choice, many, tri, commit, p +from picoparse.text import run_text_parser, as_string, quoted +from string import digits, letters +from tornado import escape, ioloop, options +from viewfinder.backend.db import db_client + + +class AttrParser(object): + """A pico parser to handle attributes specified on the command line. + Builds a map + """ + def __init__(self, table, raw=False): + self._table = table + self._raw = raw + self._updates = dict() + token_char = p(one_of, letters + digits + '.-_') + self._token = as_string(p(many, token_char)) + self._phrase = p(choice, quoted, self._token) + self._expr_parser = p(many, p(choice, p(one_of, ','), self._ParsePhrase)) + + def Run(self, attributes): + """Runs the parser on the provided comma-separated string of attributes. + """ + # Convert query_expr to Unicode since picoparser expects Unicode for non-ASCII characters. + _ = run_text_parser(self._expr_parser, escape.to_unicode(attributes)) + return self._updates + + @tri + def _ParsePhrase(self): + """Reads one attribute col_name=value. Creates a DB update + in self._updates. + """ + col_name = self._token().lower() + col_def = self._table.GetColumn(col_name) + one_of('=') + commit() + phrase = self._phrase() + if phrase: + value = eval(phrase) + if col_def.value_type == 'N': + value = int(value) + if self._raw: + self._updates[col_def.key] = db_client.UpdateAttr(value, 'PUT') + else: + self._updates[col_name] = value + else: + if self._raw: + self._updates[col_def.key] = db_client.UpdateAttr(None, 'DELETE') + else: + self._updates[col_name] = None + return None diff --git a/backend/db/user.py b/backend/db/user.py new file mode 100644 index 0000000..a4d39d0 --- /dev/null +++ b/backend/db/user.py @@ -0,0 +1,460 @@ +# Copyright 2011 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder user. + + User: viewfinder user account information +""" + +__author__ = 'spencer@emailscrubbed.com (Spencer Kimball)' + +import json + +from copy import deepcopy +from tornado import gen, web +from viewfinder.backend.base import secrets, util +from viewfinder.backend.base.exceptions import InvalidRequestError +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.analytics import Analytics +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.hash_base import DBHashObject +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.friend import Friend +from viewfinder.backend.db.id_allocator import IdAllocator +from viewfinder.backend.db.identity import Identity +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.settings import AccountSettings +from viewfinder.backend.db.subscription import Subscription +from viewfinder.backend.op.notification_manager import NotificationManager + + +@DBObject.map_table_attributes +class User(DBHashObject): + """Viewfinder user account.""" + __slots__ = [] + + # Label flags. + REGISTERED = 'registered' + """Full user that is authenticated by trusted authority and has + explicitly accepted the TOS. If this flag is not present, then the + user is a "prospective" user, which is a read-only user that has + not yet been fully registered. + """ + + STAGING = 'staging' + """Beta user that is redirected to the staging cluster. If this flag + is not present, then the user will be redirected to the main production + cluster. + """ + + TERMINATED = 'terminated' + """The account has been terminated by the user. This user can no longer + sign in, and other users can no longer share with him/her. + """ + + SYSTEM = 'system' + """System user that should not be shown in contacts even if a friend.""" + + _REGISTER_USER_ATTRIBUTES = set(['user_id', 'name', 'given_name', 'family_name', 'email', 'picture', 'gender', + 'link', 'locale', 'timezone', 'facebook_email', 'phone', 'pwd_hash', 'salt']) + """Subset of user attributes that can be set as part of user registration.""" + + _UPDATE_USER_ATTRIBUTES = set(['name', 'given_name', 'family_name', 'picture', 'pwd_hash', 'salt']) + """Subset of user attributes that can be updated by the user.""" + + _USER_FRIEND_ATTRIBUTES = set(['user_id', 'name', 'given_name', 'family_name', 'email', 'picture', 'merged_with']) + """Subset of user attributes that are visible to friends.""" + + _USER_NON_FRIEND_LABELS = set([REGISTERED, TERMINATED, SYSTEM]) + """Subset of user labels that are visible to non-friends.""" + + _USER_FRIEND_LABELS = _USER_NON_FRIEND_LABELS + """Subset of user labels that are visible to friends.""" + + _table = DBObject._schema.GetTable(vf_schema.USER) + + _ALLOCATION = 1 + _allocator = IdAllocator(id_type=_table.hash_key_col.name, allocation=_ALLOCATION) + + _RESERVED_ASSET_ID_COUNT = 1 + """Number of asset ids which are reserved for system use (default vp for now).""" + + DEFAULT_VP_ASSET_ID = 0 + """Asset id used in the default viewpoint id.""" + + def __init__(self, user_id=None): + """Creates a new user.""" + super(User, self).__init__() + self.user_id = user_id + + def IsRegistered(self): + """Returns true if this is a fully registered user that has been + authenticated by a trusted authority. + """ + return User.REGISTERED in self.labels + + def IsStaging(self): + """Returns true if this user should always be redirected to the + staging cluster. + """ + return User.STAGING in self.labels + + def IsTerminated(self): + """Returns true if this user's account has been terminated.""" + return User.TERMINATED in self.labels + + def IsSystem(self): + """Returns true if this user is a system user.""" + return User.SYSTEM in self.labels + + @gen.coroutine + def MakeSystemUser(self, client): + """Adds the SYSTEM label to this user.""" + self.labels.add(User.SYSTEM) + yield gen.Task(self.Update, client) + + def QueryIdentities(self, client, callback): + """Queries the identities (if any) attached to this user and + returns the list to the provided callback. + """ + query_str = 'identity.user_id=%d' % self.user_id + Identity.IndexQuery(client, query_str, col_names=None, callback=callback) + + @gen.coroutine + def QueryPrimaryIdentity(self, client): + """Method to return the primary identity associated with an account. + + This currently only works for prospective users, which are guaranteed to have a single + identity. The method is being created with a more general signature so that it will + be useful once the anticipated concept of a primary identity is introduced. + """ + assert not self.IsRegistered(), 'QueryPrimaryIdentity is currently only permitted for Prospective users.' + identities = yield gen.Task(self.QueryIdentities, client) + + assert len(identities) == 1, 'Encountered prospective user %d with multiple identities.' % self.user_id + raise gen.Return(identities[0]) + + @classmethod + def ShouldScrubColumn(cls, name): + """Returns list of column names that should not be printed in logs.""" + return name in ['signing_key', 'pwd_hash', 'salt'] + + def MakeLabelList(self, is_friend): + """Returns a list suitable for the 'labels' field of USER_PROFILE_METADATA. + + If 'is_friend' is True, returns labels accessible to friends; if it is false, returns + only labels accessible to strangers. + """ + if is_friend: + labels = [label for label in self.labels if label in User._USER_FRIEND_LABELS] + labels.append('friend') + return labels + else: + return [label for label in self.labels if label in User._USER_NON_FRIEND_LABELS] + + @gen.engine + def MakeUserMetadataDict(self, client, viewer_user_id, forward_friend, reverse_friend, callback): + """Projects a subset of the user attributes that can be provided to the viewing user (using + the same schema as the query_users service method). The 'forward_friend' is viewer_user_id => + friend_user_id, and the 'reverse_friend' is the reverse. This user's profile information + will only be provided to the viewer if the viewer is a reverse friend (i.e. user considers + the viewer a friend). + + The 'private' block will be returned only if viewer_user_id == self.user_id. + """ + user_dict = {'user_id': self.user_id} + + # First, populate basic user data, but only if the user considers the viewer a friend. + if reverse_friend is not None: + for attr_name in User._USER_FRIEND_ATTRIBUTES: + util.SetIfNotNone(user_dict, attr_name, getattr(self, attr_name, None)) + user_dict['labels'] = self.MakeLabelList(True) + else: + # Set labels which are visible to non-friends. + user_dict['labels'] = self.MakeLabelList(False) + + # Now project friend attributes. + for attr_name in Friend.FRIEND_ATTRIBUTES: + util.SetIfNotNone(user_dict, attr_name, getattr(forward_friend, attr_name, None)) + + # Now fill out private attributes if this user is also the viewing user. + if viewer_user_id == self.user_id: + user_dict['private'] = {} + if self.pwd_hash is None: + user_dict['private']['no_password'] = True + + subs, settings = yield [gen.Task(Subscription.QueryByUser, client, user_id=self.user_id), + gen.Task(AccountSettings.QueryByUser, client, self.user_id, None, must_exist=False)] + + sub_dicts = [sub.MakeMetadataDict() for sub in subs] + user_dict['private']['subscriptions'] = sub_dicts + + if settings is not None: + user_dict['private']['account_settings'] = settings.MakeMetadataDict() + + def _MakeIdentityDict(ident): + i_dict = {'identity': ident.key} + if ident.authority is not None: + i_dict['authority'] = ident.authority + return i_dict + + query_expr = 'identity.user_id=%d' % self.user_id + identities = yield gen.Task(Identity.IndexQuery, client, query_expr, ['key', 'authority']) + user_dict['private']['user_identities'] = [_MakeIdentityDict(ident) for ident in identities] + + callback(user_dict) + + @classmethod + def AllocateAssetIds(cls, client, user_id, num_ids, callback): + """Allocates 'num_ids' new ids from the 'asset_id_seq' column in a + block for the specified user id and returns the first id in the + sequence (first_id, ..., first_id + num_ids] with the callback + """ + id_seq_key = cls._table.GetColumn('asset_id_seq').key + + def _OnUpdateIdSeq(result): + last_id = result.return_values[id_seq_key] + first_id = last_id - num_ids + callback(first_id) + + client.UpdateItem(table=cls._table.name, + key=db_client.DBKey(hash_key=user_id, range_key=None), + attributes={id_seq_key: db_client.UpdateAttr(value=num_ids, action='ADD')}, + return_values='UPDATED_NEW', callback=_OnUpdateIdSeq) + + @classmethod + @gen.coroutine + def AllocateUserAndWebDeviceIds(cls, client): + """Allocates a new user id and a new web device id and returns them in a tuple.""" + user_id = yield gen.Task(User._allocator.NextId, client) + webapp_dev_id = yield gen.Task(Device._allocator.NextId, client) + raise gen.Return((user_id, webapp_dev_id)) + + @classmethod + @gen.engine + def QueryUsers(cls, client, viewer_user_id, user_ids, callback): + """Queries User objects for each id in the 'user_ids' list. Invokes 'callback' with a list + of (user, forward_friend, reverse_friend) tuples. Non-existent users are omitted. + """ + user_keys = [db_client.DBKey(user_id, None) for user_id in user_ids] + forward_friend_keys = [db_client.DBKey(viewer_user_id, user_id) for user_id in user_ids] + reverse_friend_keys = [db_client.DBKey(user_id, viewer_user_id) for user_id in user_ids] + users, forward_friends, reverse_friends = \ + yield [gen.Task(User.BatchQuery, client, user_keys, None, must_exist=False), + gen.Task(Friend.BatchQuery, client, forward_friend_keys, None, must_exist=False), + gen.Task(Friend.BatchQuery, client, reverse_friend_keys, None, must_exist=False)] + + user_friend_list = [] + for user, forward_friend, reverse_friend in zip(users, forward_friends, reverse_friends): + if user is not None: + user_friend_list.append((user, forward_friend, reverse_friend)) + + callback(user_friend_list) + + @classmethod + @gen.coroutine + def CreateProspective(cls, client, user_id, webapp_dev_id, identity_key, timestamp): + """Creates a prospective user with the specified user id. web device id, and identity key. + + A prospective user is typically created when photos are shared with a contact that is not + yet a Viewfinder user. + + Returns a tuple containing the user and identity. + """ + from viewfinder.backend.db.viewpoint import Viewpoint + + identity_type, identity_value = Identity.SplitKey(identity_key) + + # Ensure that identity is created. + identity = yield gen.Task(Identity.CreateProspective, + client, + identity_key, + user_id, + timestamp) + + # Create the default viewpoint. + viewpoint = yield Viewpoint.CreateDefault(client, user_id, webapp_dev_id, timestamp) + + # By default, send alerts when a new conversation is started. Send email alerts if the + # identity is email, or sms alerts if the identity is phone. + email_alerts = AccountSettings.EMAIL_ON_SHARE_NEW if identity_type == 'Email' else AccountSettings.EMAIL_NONE + sms_alerts = AccountSettings.SMS_ON_SHARE_NEW if identity_type == 'Phone' else AccountSettings.SMS_NONE + settings = AccountSettings.CreateForUser(user_id, + email_alerts=email_alerts, + sms_alerts=sms_alerts, + push_alerts=AccountSettings.PUSH_NONE) + yield gen.Task(settings.Update, client) + + # Create a Friend relation (every user is friends with himself). + friend = Friend.CreateFromKeywords(user_id=user_id, friend_id=user_id) + yield gen.Task(friend.Update, client) + + # Create the prospective user. + email = identity_value if identity_type == 'Email' else None + phone = identity_value if identity_type == 'Phone' else None + user = User.CreateFromKeywords(user_id=user_id, + private_vp_id=viewpoint.viewpoint_id, + webapp_dev_id=webapp_dev_id, + email=email, + phone=phone, + asset_id_seq=User._RESERVED_ASSET_ID_COUNT, + signing_key=secrets.CreateSigningKeyset('signing_key')) + yield gen.Task(user.Update, client) + + raise gen.Return((user, identity)) + + @classmethod + @gen.coroutine + def Register(cls, client, user_dict, ident_dict, timestamp, rewrite_contacts): + """Registers a user or updates its attributes using the contents of "user_dict". Updates + an identity using the contents of "ident_dict" and ensures its linked to the user. + + "user_dict" contains oauth-supplied user information which is either used to initially + populate the fields for a new user account, or is used to update missing fields. The + "REGISTERED" label is always added to the user object if is not yet present. + + "ident_dict" contains the identity key, authority, and various auth-specific access and + refresh tokens that will be stored with the identity. + + Returns the user object. + """ + # Create prospective user if it doesn't already exist, or else return existing user. + assert 'user_id' in user_dict, user_dict + assert 'authority' in ident_dict, ident_dict + + user = yield gen.Task(User.Query, client, user_dict['user_id'], None) + identity = yield gen.Task(Identity.Query, client, ident_dict['key'], None) + + # Update user attributes (only if they have not yet been set). + for k, v in user_dict.items(): + assert k in User._REGISTER_USER_ATTRIBUTES, user_dict + if getattr(user, k) is None: + setattr(user, k, v) + + # Ensure that prospective user is registered. + user.labels.add(User.REGISTERED) + + if rewrite_contacts: + yield identity._RewriteContacts(client, timestamp) + + yield gen.Task(user.Update, client) + + # Update identity attributes. + assert identity.user_id is None or identity.user_id == user.user_id, (identity, user) + identity.user_id = user.user_id + identity.UpdateFromKeywords(**ident_dict) + yield gen.Task(identity.Update, client) + + raise gen.Return(user) + + @classmethod + @gen.engine + def UpdateWithSettings(cls, client, user_dict, settings_dict, callback): + """Update the user's public profile, as well as any account settings specified in + "settings_dict". + """ + # Update user profile attributes. + assert all(attr_name == 'user_id' or attr_name in User._UPDATE_USER_ATTRIBUTES + for attr_name in user_dict), user_dict + + # If any name attribute is updated, update them all, if only to None. This helps prevent + # accidental divergence of name from given_name/family_name. + for attr_name in ['name', 'given_name', 'family_name']: + if attr_name in user_dict: + user_dict.setdefault('name', None) + user_dict.setdefault('given_name', None) + user_dict.setdefault('family_name', None) + break + + user = yield gen.Task(User.Query, client, user_dict['user_id'], None) + user.UpdateFromKeywords(**user_dict) + yield gen.Task(user.Update, client) + + if settings_dict: + # Add keys to dict. + scratch_settings_dict = deepcopy(settings_dict) + scratch_settings_dict['settings_id'] = AccountSettings.ConstructSettingsId(user_dict['user_id']) + scratch_settings_dict['group_name'] = AccountSettings.GROUP_NAME + scratch_settings_dict['user_id'] = user_dict['user_id'] + + # Update any user account settings. + settings = yield gen.Task(AccountSettings.QueryByUser, client, user_dict['user_id'], None, must_exist=False) + if settings is None: + settings = AccountSettings.CreateFromKeywords(**scratch_settings_dict) + else: + settings.UpdateFromKeywords(**scratch_settings_dict) + yield gen.Task(settings.Update, client) + + callback() + + @classmethod + @gen.coroutine + def TerminateAccount(cls, client, user_id, merged_with): + """Terminate the user's account by adding the "TERMINATED" flag to the labels set. If + "merged_with" is not None, then the terminate is due to a merge, so set the "merged_with" + field on the terminated user. + """ + user = yield gen.Task(User.Query, client, user_id, None) + user.merged_with = merged_with + user.labels.add(User.TERMINATED) + yield gen.Task(user.Update, client) + + @classmethod + def CreateUnsubscribeCookie(cls, user_id, email_type): + """Create a user unsubscribe cookie that is passed as an argument to the unsubscribe handler, + and which proves control of the given user id. + """ + unsubscribe_dict = {'user_id': user_id, 'email_type': email_type} + return web.create_signed_value(secrets.GetSecret('invite_signing'), 'unsubscribe', json.dumps(unsubscribe_dict)) + + @classmethod + def DecodeUnsubscribeCookie(cls, unsubscribe_cookie): + """Decode a user unsubscribe cookie that is passed as an argument to the unsubscribe handler. + Returns the unsubscribe dict containing the user_id and email_type originally passed to + CreateUnsubscribeCookie. + """ + value = web.decode_signed_value(secrets.GetSecret('invite_signing'), + 'unsubscribe', + unsubscribe_cookie) + return None if value is None else json.loads(value) + + @classmethod + @gen.coroutine + def TerminateAccountOperation(cls, client, user_id, merged_with=None): + """Invokes User.TerminateAccount via operation execution.""" + @gen.coroutine + def _VisitIdentity(identity_key): + """Unlink this identity from the user.""" + yield Identity.UnlinkIdentityOperation(client, user_id, identity_key.hash_key) + + # Turn off alerts to all devices owned by the user. + yield gen.Task(Device.MuteAlerts, client, user_id) + + # Unlink every identity attached to the user. + query_expr = ('identity.user_id={id}', {'id': user_id}) + yield gen.Task(Identity.VisitIndexKeys, client, query_expr, _VisitIdentity) + + # Add an analytics entry for this user. + timestamp = Operation.GetCurrent().timestamp + payload = 'terminate' if merged_with is None else 'merge=%s' % merged_with + analytics = Analytics.Create(entity='us:%d' % user_id, + type=Analytics.USER_TERMINATE, + timestamp=timestamp, + payload=payload) + yield gen.Task(analytics.Update, client) + + # Terminate the user account. + yield gen.Task(User.TerminateAccount, client, user_id, merged_with=merged_with) + + # Notify all friends that this user account has been terminated. + yield NotificationManager.NotifyTerminateAccount(client, user_id) + + @classmethod + @gen.engine + def UpdateOperation(cls, client, callback, user_dict, settings_dict): + """Invokes User.Update via operation execution.""" + yield gen.Task(User.UpdateWithSettings, client, user_dict, settings_dict) + + timestamp = Operation.GetCurrent().timestamp + yield NotificationManager.NotifyUpdateUser(client, user_dict, settings_dict, timestamp) + + callback() diff --git a/backend/db/user_photo.py b/backend/db/user_photo.py new file mode 100644 index 0000000..8ca828e --- /dev/null +++ b/backend/db/user_photo.py @@ -0,0 +1,74 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved +"""UserPhoto data object.""" + +__author__ = 'ben@emailscrubbed.com (Ben Darnell)' + +from tornado import gen + +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject +from viewfinder.backend.db import versions + +@DBObject.map_table_attributes +class UserPhoto(DBRangeObject): + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.USER_PHOTO) + + @classmethod + def AssetKeyToFingerprint(cls, asset_key): + """Converts an asset key to a fingerprint-only asset key. + + Asset keys from the client may (in 1.4) contain asset urls that + are only meaningful for that device. We only want to store + the fingerprint portion of the asset key. + + If the asset key does not contain a fingerprint, returns None. + """ + # See DecodeAssetKey() in PhotoTable.mm + if asset_key.startswith('a/'): + _, sep, fingerprint = asset_key.rpartition('#') + if sep and fingerprint: + return 'a/#' + fingerprint + return None + + @classmethod + def MakeAssetFingerprintSet(cls, asset_keys): + fingerprints = set(UserPhoto.AssetKeyToFingerprint(k) for k in asset_keys) + # If any keys were missing fingerprints, discard the None entry. + fingerprints.discard(None) + return fingerprints + + def MergeAssetKeys(self, new_keys): + """Merges the asset keys in new_keys into self.asset_keys. + + Returns True if any changes were made. + """ + changed = False + for key in new_keys: + fingerprint = UserPhoto.AssetKeyToFingerprint(key) + if fingerprint is not None and fingerprint not in self.asset_keys: + self.asset_keys.add(fingerprint) + changed = True + return changed + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **up_dict): + asset_keys = up_dict.pop('asset_keys', []) + up = UserPhoto.CreateFromKeywords(**up_dict) + up.MergeAssetKeys(asset_keys) + yield gen.Task(up.Update, client) + + @classmethod + @gen.coroutine + def UpdateOperation(cls, client, up_dict): + asset_keys = up_dict.pop('asset_keys', []) + user_photo = yield gen.Task(UserPhoto.Query, client, up_dict['user_id'], up_dict['photo_id'], None, must_exist=False) + if user_photo is None: + user_photo = UserPhoto.CreateFromKeywords(**up_dict) + else: + yield gen.Task(versions.Version.MaybeMigrate, client, user_photo, [versions.REMOVE_ASSET_URLS]) + user_photo.MergeAssetKeys(asset_keys) + yield user_photo.Update(client) diff --git a/backend/db/user_post.py b/backend/db/user_post.py new file mode 100644 index 0000000..466c1cf --- /dev/null +++ b/backend/db/user_post.py @@ -0,0 +1,46 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""UserPost relation. + +By default, users have a relationship with photos that are contained +within viewpoints they follow. By default, all photos contained in +non-public viewpoints are shown as part of the user's personal +collection. However, the user may want to hide certain of these photos +from view. Inversely, the user may want to show public photos in his +personal view. This relation allows these kinds of per-user +customization of photos. + +Each photo can be stamped with "labels" which describe the user- +specific customizations made to that photo. The name of each label +is chosen so that "is " makes sense. + + 'removed': the photo should not be shown in the user's personal + collection, and will ultimately deleted from the + server if no other references to it are held. +""" + +__author__ = 'andy@emailscrubbed.com (Andy Kimball)' + +import logging +import time + +from functools import partial +from tornado import gen +from viewfinder.backend.db import vf_schema +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.range_base import DBRangeObject + +@DBObject.map_table_attributes +class UserPost(DBRangeObject): + """UserPost data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.USER_POST) + + HIDDEN = 'hidden' + + def IsHidden(self): + """Returns true if the photo has been hidden by the user so that it will not show in the + personal library or conversation feed. + """ + return UserPost.HIDDEN in self.labels diff --git a/backend/db/versions.py b/backend/db/versions.py new file mode 100644 index 0000000..3315ae0 --- /dev/null +++ b/backend/db/versions.py @@ -0,0 +1,1115 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Feature versions and data migration. + +Versions describe how new features or functionality should change the +data in a row (aka an item) of the table. For example, new columns +might be added, old columns deprecated, values transformed, the +indexing algorithm changed, etc. Each version 'tag' refers to a +function which makes the relevant changes to the an item. The item can +then be updated in the table with its version # updated to the number +of the tag. This mechanism allows a sequential scan (as implemented in +backend/db/tools/upgrade.py) to migrate all rows in a table. It also +allows changes to be made lazily, where items are upgraded in +arbitrary batches. Any new features not reflected in the item's +version # are applied in order, automatically. If the item is then +updated, the new version # and the transformations are preserved, but +this isn't necessary. + +Some features, such as a change in indexing, should be migrated in toto +via db/tools/upgrade. Other features, such as deprecating a column, +can be done piecemeal, with items which are never accessed being updated +only on the next full table migration. + +The ordinality of features is important as any set of transformations +which must take place on a row may have ordered dependencies. +""" + +import json +import logging +import time + +from functools import partial +from tornado import gen, httpclient +from viewfinder.backend.base import message, util, retry, secrets +from viewfinder.backend.base.context_local import ContextLocal + +class Version(object): + """The version base class. Provides a guarantee that rank order is + correct (that careless code updating doesn't confuse the ordering of + versions). + + When authoring a version migrator, do not assume that the version + is correct. There are cases where it can "get behind". So, for + example, while the version on the row = 5, the actual row data + corresponds to version 8. However, the reverse *cannot* happen; + if the version = 8, then the actual row data cannot correspond + to version 5. So if a row version shows the row is at the latest + version, then there is no need to migrate. + """ + _DELETED_MIGRATOR_COUNT = 16 + """If older migrators are deleted, update this.""" + + _version_classes = set() + _rank_ordering = [] + _mutate_items = True + _allow_s3_queries = True + + _migrate_retry_policy = retry.RetryPolicy(max_tries=5, min_delay=1, + check_exception=lambda type, value, tb: True) + """Retry migration of an item up to 5 times before giving up.""" + + def __init__(self): + self.rank = len(Version._rank_ordering) + 1 + Version._DELETED_MIGRATOR_COUNT + if Version._rank_ordering: + assert self.rank > Version._rank_ordering[-1], \ + 'rank is out of order (! %d > %d)' % (self.rank, Version._rank_ordering[-1]) + assert self.__class__ not in Version._version_classes, \ + 'class %s has already been added to version set' % self.__class__.__name__ + Version._rank_ordering.append(self.rank) + Version._version_classes.add(self.__class__) + + @classmethod + def SetMutateItems(cls, mutate): + """Set to affect mutations on the database. If False, the planned + modifications to each item are verbosely logged but not persisted. + """ + Version._mutate_items = mutate + + @classmethod + def SetAllowS3Queries(cls, allow): + """Allow S3 queries. If False, upgrades that involve querying S3 will + skip it, but may perform other work. + eg: CreateMD5Hashes and FillFileSizes both use S3 queries as a fallback + when the desired fields are not found the Photo.client_data. + """ + Version._allow_s3_queries = allow + + @classmethod + def GetCurrentVersion(cls): + """Returns the maximum version. New objects have item._version set + to this value. + """ + if Version._rank_ordering: + return Version._rank_ordering[-1] + else: + return 0 + + @classmethod + def MaybeMigrate(cls, client, original_item, versions, callback): + """Migrates the data in one table row ('item') by advancing + 'item's version via successive data migrations. If 'item' does not + have a version yet, all data migrations are applied. If item's + version is current, does nothing. Return the migrated object if + mutations are enabled, or the original object if not. Take care + if the migration changes the primary key; the caller might fetch + an object using one primary key, but get back an object with a + different migrated primary key! + """ + def _Migrate(start_rank, mutate_item): + last_rank = 0 + for version in versions: + if version.rank < start_rank: + last_rank = version.rank + continue + assert version.rank > last_rank, \ + 'tags listed out of order (! %d > %d)' % (version.rank, last_rank) + last_rank = version.rank + item_version = mutate_item._version or 0 + if item_version < version.rank: + logging.debug('upgrading item from %s to version %s' % + (type(mutate_item)._table.name, version.__class__.__name__)) + mutate_item._version = version.rank + + # If Transform fails, retry several times before giving up. + transform_callback = partial(_Migrate, last_rank + 1) + retry.CallWithRetryAsync(Version._migrate_retry_policy, + version.Transform, + client, + mutate_item, + callback=transform_callback) + return + + callback(mutate_item if Version._mutate_items else original_item) + + _Migrate(0, original_item if Version._mutate_items else original_item._Clone()) + + def _LogUpdate(self, item): + """Log the changes to the object.""" + mods = ['%s => %r' % (n, getattr(item, n)) for n in item.GetColNames() if item._IsModified(n)] + if mods: + logging.info('%s (%r): %s' % (type(item)._table.name, item.GetKey(), ', '.join(mods))) + + def Transform(self, client, item, callback): + """Implement in each subclass to effect the required data migration. + 'callback' should be invoked on completion with the update object. + If no async processing is required, it should be invoked directly. + """ + raise NotImplementedError() + + +class TestVersion(Version): + """Upgrade rows in the Test table.""" + def Transform(self, client, item, callback): + """If an attribute does not exist, create it with default value.""" + # Test creating brand new object. + item = item._Clone() + if item.attr0 is None: + item.attr0 = 100 + self._LogUpdate(item) + if Version._mutate_items: + item.Update(client, partial(callback, item)) + else: + callback(item) + + +class TestVersion2(Version): + """Upgrade rows in the Test table.""" + def Transform(self, client, item, callback): + """If an attribute does exist, delete it.""" + if item.attr1 is not None: + item.attr1 = None + self._LogUpdate(item) + if Version._mutate_items: + item.Update(client, partial(callback, item)) + else: + callback(item) + + +class AddViewpointSeq(Version): + """Add update_seq attribute to Viewpoint and viewed_seq attribute to + Follower, each with starting value of 0. + """ + def Transform(self, client, follower, callback): + from viewpoint import Viewpoint + + def _OnQuery(viewpoint): + with util.Barrier(partial(callback, follower)) as b: + follower.viewed_seq = 0 + self._LogUpdate(follower) + + if Version._mutate_items: + follower.Update(client, b.Callback()) + + if viewpoint.update_seq is None: + viewpoint._version = self.rank + viewpoint.update_seq = 0 + self._LogUpdate(viewpoint) + + if Version._mutate_items: + viewpoint.Update(client, b.Callback()) + + Viewpoint.Query(client, follower.viewpoint_id, None, _OnQuery) + + +class AddActivitySeq(Version): + """Add update_seq attribute to existing activities, with starting + value of 0. + """ + def Transform(self, client, activity, callback): + activity.update_seq = 0 + self._LogUpdate(activity) + + if Version._mutate_items: + activity.Update(client, partial(callback, activity)) + else: + callback(activity) + + +class UpdateActivityShare(Version): + """Add follower_ids to share activities and rename to either share_new + or share_existing, depending on whether the activity's timestamp is + before all other activities in the viewpoint. + """ + def Transform(self, client, viewpoint, callback): + from activity import Activity + from viewpoint import Viewpoint + + def _OnQuery(followers_activities): + (follower_ids, last_key), activities = followers_activities + + activities = [activity for activity in activities if activity.name == 'share'] + + if len(activities) > 0: + # Find the share activity with the lowest timestamp. + oldest_activity = None + for activity in activities: + if oldest_activity is None or activity.timestamp < oldest_activity.timestamp: + oldest_activity = activity + activity.name = 'share_existing' + + # Override oldest activity as share_new and add followers. + oldest_activity.name = 'share_new' + act_dict = json.loads(activities[-1].json) + act_dict['follower_ids'] = [f_id for f_id in follower_ids if f_id != viewpoint.user_id] + oldest_activity.json = json.dumps(act_dict) + + # Update all activities. + with util.Barrier(partial(callback, viewpoint)) as b: + for activity in activities: + self._LogUpdate(activity) + + if Version._mutate_items: + activity.Update(client, b.Callback()) + else: + b.Callback()() + else: + callback(viewpoint) + + with util.ArrayBarrier(_OnQuery) as b: + Viewpoint.QueryFollowerIds(client, viewpoint.viewpoint_id, b.Callback()) + Activity.RangeQuery(client, viewpoint.viewpoint_id, None, None, None, b.Callback()) + + +class CreateMD5Hashes(Version): + """Create tn_md5 and med_md5 attributes on all photos by extracting + the hashes from the client_data attribute, if possible. If they don't + exist there, then get them by issuing HEAD requests against S3. As a + side effect, this upgrade will also fix the placemark base64hex padding + issue. + """ + def Transform(self, client, photo, callback): + from viewfinder.backend.storage.s3_object_store import S3ObjectStore + + def _SetPhotoMD5Values(md5_values): + tn_md5, med_md5, full_md5, orig_md5 = md5_values + + assert photo.tn_md5 == tn_md5 or photo.tn_md5 is None, photo + photo.tn_md5 = tn_md5 + + assert photo.med_md5 == med_md5 or photo.med_md5 is None, photo + photo.med_md5 = med_md5 + + assert photo.full_md5 == full_md5 or photo.full_md5 is None, photo + photo.full_md5 = full_md5 + + assert photo.orig_md5 == orig_md5 or photo.orig_md5 is None, photo + photo.orig_md5 = orig_md5 + + photo.placemark = photo.placemark + + self._LogUpdate(photo) + + if Version._mutate_items: + photo.Update(client, partial(callback, photo)) + else: + callback(photo) + + def _OnFetchHead(head_callback, response): + # Etag is the hex string encoded MD5 hash of the photo. + if response.code != 404: + etag = response.headers['Etag'][1:-1] + else: + etag = None + head_callback(etag) + + def _SendHeadRequest(photo_id, suffix, head_callback): + object_store = S3ObjectStore('photos-viewfinder-co') + url = object_store.GenerateUrl(photo_id + suffix, method='HEAD') + http_client = httpclient.AsyncHTTPClient() + http_client.fetch(url, method='HEAD', callback=partial(_OnFetchHead, head_callback)) + + client_data = photo.client_data + if client_data is None or 'tn_md5' not in client_data: + if not Version._allow_s3_queries: + callback(photo) + return + # Get MD5 values by issuing HEAD against S3. + with util.ArrayBarrier(_SetPhotoMD5Values) as b: + _SendHeadRequest(photo.photo_id, '.t', b.Callback()) + _SendHeadRequest(photo.photo_id, '.m', b.Callback()) + _SendHeadRequest(photo.photo_id, '.f', b.Callback()) + _SendHeadRequest(photo.photo_id, '.o', b.Callback()) + else: + # Get MD5 values by extracting from client_data. + _SetPhotoMD5Values((client_data['tn_md5'], client_data['med_md5'], + photo.full_md5, photo.orig_md5)) + + +class DisambiguateActivityIds(Version): + """Search for activities which were migrated from old data model to + new data model. In some cases, the assigned activity ids are duplicated + across viewpoints. This happened when an episode "sharing tree" involved + multiple users. For each unique pair of users who had access to the + episode, an activity with the same activity id was created. This upgrade + appends the viewpoint id to the activity id to make it globally unique + (rather than just unique within a particular viewpoint). + """ + pass + + +class MigrateMoreShares(Version): + """Migrate shares which were missed during the last migration because + the member sharing_user_id attribute was not specified. Infer the + attribute by assuming that the photo owner was the sharer. + """ + def Transform(self, client, member, callback): + from episode import Episode + from member import Member + + def _OnQueryMember(root_episode, sharer_member): + assert Member.OWNED in sharer_member.labels, sharer_member + + logging.info('migrating share from user "%s" to user "%s" in episode "%s" in viewpoint "%s"' % \ + (sharer_member.user_id, member.user_id, root_episode.episode_id, + root_episode.viewpoint_id)) + + if Version._mutate_items: + Episode._MigrateShare(client, root_episode, sharer_member=sharer_member, + recipient_member=member, add_photo_ids=None, + callback=partial(callback, member)) + else: + callback(member) + + def _OnQueryEpisode(root_episode): + Member.Query(client, root_episode.user_id, root_episode.episode_id, None, + partial(_OnQueryMember, root_episode)) + + if member.sharing_user_id is None and Member.OWNED not in member.labels: + assert list(member.labels) == [Member.SHARED], member + Episode.Query(client, member.episode_id, None, _OnQueryEpisode) + else: + callback(member) + + +class AddUserType(Version): + """Set user "type" attribute to "activated", and also get rid of the + "op_id_seq" attribute, which is no longer used. + """ + def Transform(self, client, user, callback): + from user import User + + user.labels = [User.ACTIVATED] + user.op_id_seq = None + + self._LogUpdate(user) + + if Version._mutate_items: + user.Update(client, partial(callback, user)) + else: + callback(user) + + +class DisambiguateActivityIds2(Version): + """Search for activities which were migrated from old data model to + new data model. In some cases, the assigned activity ids are duplicated + across viewpoints. This happened when an episode "sharing tree" involved + multiple users. For each unique pair of users who had access to the + episode, an activity with the same activity id was created. This upgrade + appends the viewpoint id to the activity id to make it globally unique + (rather than just unique within a particular viewpoint). + + NOTE: This is being done a second time, as we've discovered a new case + in which activity-id dups have been created. + """ + _unique_activity_ids = set() + + def Transform(self, client, activity, callback): + from activity import Activity + from asset_id import AssetIdUniquifier + from device import Device + + def _OnUpdate(new_activity): + """Delete the old activity.""" + if Version._mutate_items: + activity.Delete(client, partial(callback, new_activity)) + else: + callback(new_activity) + + timestamp, device_id, uniquifier = Activity.DeconstructActivityId(activity.activity_id) + + if device_id == Device.SYSTEM: + if activity.activity_id in DisambiguateActivityIds2._unique_activity_ids: + # Already saw this activity id, so append viewpoint id to it. + assert uniquifier.server_id is None, (activity, uniquifier) + new_uniquifier = AssetIdUniquifier(uniquifier.client_id, activity.viewpoint_id) + new_activity_id = Activity.ConstructActivityId(timestamp, device_id, new_uniquifier) + + new_activity_dict = activity._asdict() + new_activity_dict['activity_id'] = new_activity_id + + new_activity = Activity.CreateFromKeywords(**new_activity_dict) + + logging.info('%s\n%s (%s/%s/%s) => %s (%s/%s/%s/%s)' % + (activity, activity.activity_id, timestamp, device_id, uniquifier.client_id, + new_activity_id, timestamp, device_id, new_uniquifier.client_id, new_uniquifier.server_id)) + + if Version._mutate_items: + new_activity.Update(client, partial(_OnUpdate, new_activity)) + else: + _OnUpdate(new_activity) + else: + DisambiguateActivityIds2._unique_activity_ids.add(activity.activity_id) + callback(activity) + else: + assert activity.activity_id not in DisambiguateActivityIds2._unique_activity_ids, activity + callback(activity) + + +class CopyUpdateSeq(Version): + """Copy the update_seq column from the activity table to the notification + table. + """ + def Transform(self, client, notification, callback): + from activity import Activity + from asset_id import AssetIdUniquifier + + def _DoUpdate(): + self._LogUpdate(notification) + + if Version._mutate_items: + notification.Update(client, partial(callback, notification)) + else: + callback(notification) + + def _OnQueryNewActivity(activity): + if activity is None: + logging.warning('notification does not have a valid activity_id: %s', notification) + notification.update_seq = 0 + self._LogUpdate(notification) + _DoUpdate() + else: + _OnQueryActivity(activity) + + def _OnQueryActivity(activity): + if activity is None: + # Also migrate any notifications which are using the older activity id. + timestamp, device_id, uniquifier = Activity.DeconstructActivityId(notification.activity_id) + new_uniquifier = AssetIdUniquifier(uniquifier.client_id, notification.viewpoint_id) + new_activity_id = Activity.ConstructActivityId(timestamp, device_id, new_uniquifier) + Activity.Query(client, notification.viewpoint_id, new_activity_id, None, + _OnQueryNewActivity, must_exist=False) + return + + notification.activity_id = activity.activity_id + notification.update_seq = activity.update_seq + _DoUpdate() + + if notification.activity_id is None: + callback(notification) + else: + Activity.Query(client, notification.viewpoint_id, notification.activity_id, None, + _OnQueryActivity, must_exist=False) + + +class AddUserSigningKey(Version): + """Add a Keyczar signing keyset to each User db object.""" + def Transform(self, client, user, callback): + user.signing_key = secrets.CreateSigningKeyset('signing_key') + self._LogUpdate(user) + + if Version._mutate_items: + user.Update(client, partial(callback, user)) + else: + callback(user) + + +class UpdateUserType(Version): + """Update user "type" attribute from "activated" to "registered". + """ + def Transform(self, client, user, callback): + from user import User + + if 'activated' in user.labels: + labels = user.labels.combine() + labels.remove('activated') + labels.add(User.REGISTERED) + user.labels = labels + + self._LogUpdate(user) + + if Version._mutate_items: + user.Update(client, partial(callback, user)) + else: + callback(user) + + +class AddFollowed(Version): + """Add one record to "Followed" table for each viewpoint. The records + are sorted by the timestamp of the latest activity in each viewpoint. + """ + def Transform(self, client, viewpoint, callback): + from activity import Activity + from followed import Followed + from viewpoint import Viewpoint + + def _OnQuery(activities_followers): + activities, (follower_ids, last_key) = activities_followers + + with util.Barrier(partial(callback, viewpoint)) as b: + old_timestamp = viewpoint.last_updated + + if len(activities) > 0: + new_timestamp = max(a.timestamp for a in activities) + else: + # Viewpoint has no activities. + new_timestamp = 0 + + viewpoint.last_updated = new_timestamp + self._LogUpdate(viewpoint) + + for follower_id in follower_ids: + logging.info('Followed (user_id=%s, viewpoint_id=%s): %s => %s' % + (follower_id, viewpoint.viewpoint_id, Followed._TruncateToDay(old_timestamp), + Followed._TruncateToDay(new_timestamp))) + + if Version._mutate_items: + Followed.UpdateDateUpdated(client, follower_id, viewpoint.viewpoint_id, + old_timestamp, new_timestamp, b.Callback()) + + with util.ArrayBarrier(_OnQuery) as b: + Activity.RangeQuery(client, viewpoint.viewpoint_id, None, None, None, b.Callback()) + Viewpoint.QueryFollowerIds(client, viewpoint.viewpoint_id, b.Callback()) + + +class InTransformContext(ContextLocal): + """ContextLocal used to prevent transform re-entrancy.""" + pass + + +class UpdateDevices(Version): + """Add columns and indexes to devices and remove duplicate and + expired push tokens. + """ + def Transform(self, client, device, callback): + from tornado.web import stack_context + from viewfinder.backend.db.device import Device + + def _DoUpdate(): + self._LogUpdate(device) + + if Version._mutate_items: + device.Update(client, partial(callback, device)) + else: + callback(device) + + def _OnQueryByPushToken(other_devices): + # If another device with the same push token and with a greater id exists, then erase this + # device's push token. + for other in other_devices: + if other.push_token == device.push_token and other.device_id > device.device_id: + device.alert_user_id = None + device.push_token = None + break + + _DoUpdate() + + # Do not allow re-entrancy, which is caused by the Device.IndexQuery below. Put a + # ContextLocal into scope so that re-entrancy can be detected. + if InTransformContext.current() is None: + with stack_context.StackContext(InTransformContext()): + # Set the new device timestamp field. + device.timestamp = device.last_access or time.time() + + if device.push_token is not None: + # Enable alerts if the device is still active. + if time.time() < device.last_access + Device._PUSH_EXPIRATION: + device.alert_user_id = device.user_id + + # Find all devices with the same push token. + query_expr = ('device.push_token={t}', {'t': device.push_token}) + Device.IndexQuery(client, query_expr, None, _OnQueryByPushToken) + else: + _DoUpdate() + else: + callback(device) + +class FillFileSizes(Version): + """Fill in the file size columns (tn/med/full/orig) from the client_data.""" + def Transform(self, client, photo, callback): + from viewfinder.backend.storage.s3_object_store import S3ObjectStore + + def _SetPhotoSizeValues(size_values): + tn_size, med_size, full_size, orig_size = size_values + + assert photo.tn_size == tn_size or photo.tn_size is None, photo + photo.tn_size = tn_size + + assert photo.med_size == med_size or photo.med_size is None, photo + photo.med_size = med_size + + assert photo.full_size == full_size or photo.full_size is None, photo + photo.full_size = full_size + + assert photo.orig_size == orig_size or photo.orig_size is None, photo + photo.orig_size = orig_size + + self._LogUpdate(photo) + + if Version._mutate_items: + photo.Update(client, partial(callback, photo)) + else: + callback(photo) + + def _OnListKeys(result): + def _GetSizeOrNone(files, suffix): + fname = photo.photo_id + '.' + suffix + if fname in files: + return int(files[fname]['Size']) + else: + return None + + if len(result) > 0: + _SetPhotoSizeValues((_GetSizeOrNone(result, 't'), _GetSizeOrNone(result, 'm'), + _GetSizeOrNone(result, 'f'), _GetSizeOrNone(result, 'o'))) + else: + logging.info('No files found in S3 for photo %s' % photo.photo_id) + callback(photo) + + def _ListPhotoMetadata(): + object_store = S3ObjectStore('photos-viewfinder-co') + object_store.ListKeyMetadata(_OnListKeys, prefix=photo.photo_id + '.', + fields=['Size', 'Key']) + + client_data = photo.client_data + # the assumption is that is we have the tn_size field set, all other + # sizes will be as well. db analysis shows this to be currently true. + if client_data is None or 'tn_size' not in client_data: + if not Version._allow_s3_queries: + callback(photo) + return + # Extract sizes from S3. + _ListPhotoMetadata() + else: + # Extract sizes from client_data. + _SetPhotoSizeValues((int(client_data['tn_size']), int(client_data['med_size']), + int(client_data['full_size']), int(client_data['orig_size']))) + +class RepairFacebookContacts(Version): + @gen.engine + def Transform(self, client, identity, callback): + from tornado.web import stack_context + from operation import Operation + from user import User + + # Do not allow re-entrancy, which is caused by the Device.IndexQuery below. Put a + # ContextLocal into scope so that re-entrancy can be detected. + if InTransformContext.current() is None: + with stack_context.StackContext(InTransformContext()): + if identity.authority == 'Facebook' and identity.user_id is not None: + user = yield gen.Task(User.Query, client, identity.user_id, None) + yield gen.Task(Operation.CreateAndExecute, client, identity.user_id, user.webapp_dev_id, + 'FetchContactsOperation.Execute', + {'headers': {'synchronous': True}, 'key': identity.key, 'user_id': identity.user_id}) + + callback(identity) + +class AddAccountSettings(Version): + """Add account settings to every existing user, defaulting to full push notifications.""" + @gen.engine + def Transform(self, client, user, callback): + from settings import AccountSettings + + settings = yield gen.Task(AccountSettings.QueryByUser, client, user.user_id, None, must_exist=False) + if settings is None: + logging.info('Creating account settings for user %d', user.user_id) + settings = AccountSettings.CreateForUser(user.user_id, + email_alerts=AccountSettings.EMAIL_NONE, + push_alerts=AccountSettings.PUSH_ALL) + if Version._mutate_items: + yield gen.Task(settings.Update, client) + + callback(user) + +class SplitUserNames(Version): + """Split full names of all users that don't have given/family names specified.""" + @gen.engine + def Transform(self, client, user, callback): + if user.name and '@' not in user.name and not user.given_name and not user.family_name: + match = message.FULL_NAME_RE.match(user.name) + if match is not None: + user.given_name = match.group(1) + if match.group(2): + user.family_name = match.group(2) + + self._LogUpdate(user) + + if Version._mutate_items: + yield gen.Task(user.Update, client) + + callback(user) + +class ExtractAssetKeys(Version): + @gen.engine + def Transform(self, client, photo, callback): + from device_photo import DevicePhoto + from photo import Photo + + if photo.client_data is None: + callback(photo) + return + + if photo.client_data.get('asset_key'): + # Asset keys were only stored in client_data for the device that originally uploaded the photo. + _, device_id, _ = Photo.DeconstructPhotoId(photo.photo_id) + existing = yield gen.Task(DevicePhoto.Query, client, device_id, photo.photo_id, None, must_exist=False) + if existing is None: + logging.info('Creating device photo for photo %s, device %s', photo.photo_id, device_id) + device_photo = DevicePhoto.CreateFromKeywords(photo_id=photo.photo_id, + device_id=device_id, + asset_keys=[photo.client_data['asset_key']]) + else: + logging.info('Photo %s, device %s already has device photo', photo.photo_id, device_id) + device_photo = None + + if device_photo is not None: + self._LogUpdate(device_photo) + photo.client_data = None + self._LogUpdate(photo) + + if Version._mutate_items: + # Do this serially with device_photo first so we can retry on partial failure. + if device_photo is not None: + yield gen.Task(device_photo.Update, client) + yield gen.Task(photo.Update, client) + + callback(photo) + +class MyOnlyFriend(Version): + """Make each user a friend with himself/herself.""" + @gen.engine + def Transform(self, client, user, callback): + from friend import Friend + friend = Friend.CreateFromKeywords(user_id=user.user_id, friend_id=user.user_id) + + logging.info('Creating friend for user %d', user.user_id) + if Version._mutate_items: + yield gen.Task(friend.Update, client) + + callback(user) + +class EraseFriendNames(Version): + """Remove the name attributes from all friends.""" + @gen.engine + def Transform(self, client, friend, callback): + + friend.name = None + self._LogUpdate(friend) + if Version._mutate_items: + yield gen.Task(friend.Update, client) + + callback(friend) + + +class MoveDevicePhoto(Version): + _device_to_user_cache = {} + + @gen.engine + def Transform(self, client, device_photo, callback): + from device import Device + from user_photo import UserPhoto + device_id = device_photo.device_id + + if device_id not in MoveDevicePhoto._device_to_user_cache: + query_expr = ('device.device_id={t}', {'t': device_id}) + devices = yield gen.Task(Device.IndexQuery, client, query_expr, None) + assert len(devices) == 1 + MoveDevicePhoto._device_to_user_cache[device_id] = devices[0].user_id + user_id = MoveDevicePhoto._device_to_user_cache[device_id] + + existing = yield gen.Task(UserPhoto.Query, client, user_id, device_photo.photo_id, None, must_exist=False) + if existing is None: + logging.info('Creating user photo for photo %s, device %s, user %s', device_photo.photo_id, device_id, user_id) + user_photo = UserPhoto.CreateFromKeywords(photo_id=device_photo.photo_id, + user_id=user_id, + asset_keys=device_photo.asset_keys) + else: + logging.info('Photo %s, device %s, user %s already has user photo', device_photo.photo_id, device_id, user_id) + user_photo = None + + if user_photo is not None: + self._LogUpdate(user_photo) + + if Version._mutate_items and user_photo is not None: + yield gen.Task(user_photo.Update, client) + + callback(device_photo) + +class SetCoverPhoto(Version): + """Set a cover photo on each viewpoint. + This will use the original mobile client algorithm to select the cover photo. + """ + @gen.engine + def Transform(self, client, viewpoint, callback): + # We don't set a cover_photo on DEFAULT viewpoints. + if not viewpoint.IsDefault() and viewpoint.cover_photo == None: + viewpoint.cover_photo = yield gen.Task(viewpoint.SelectCoverPhotoUsingOriginalAlgorithm, client) + if Version._mutate_items: + yield gen.Task(viewpoint.Update, client) + + callback(viewpoint) + +class RemoveAssetUrls(Version): + """Remove the asset urls from client-supplied asset keys, leaving only + the fingerprints. + """ + @gen.engine + def Transform(self, client, user_photo, callback): + from user_photo import UserPhoto + asset_keys = user_photo.asset_keys.combine() + changed = False + # Copy the set because we'll modify it as we go. + for asset_key in list(asset_keys): + fingerprint = UserPhoto.AssetKeyToFingerprint(asset_key) + if fingerprint is None: + # Old asset key with no fingerprint; throw it away. + asset_keys.remove(asset_key) + changed = True + elif asset_key != fingerprint: + # A url was present, remove it and just leave the fingerprint. + asset_keys.remove(asset_key) + asset_keys.add(fingerprint) + changed = True + if changed: + # Set columns don't support adding and deleting values in the same + # operation unless you delete and rewrite the whole thing. + # That's not entirely atomic, but the risk here is low so it's + # not worth the trouble of splitting the adds and deletes into separate + # phases. + user_photo.asset_keys = asset_keys + + self._LogUpdate(user_photo) + if Version._mutate_items: + yield gen.Task(user_photo.Update, client) + + callback(user_photo) + +class RemoveContactUserId(Version): + """Remove the contact_user_id field from contact records.""" + @gen.engine + def Transform(self, client, contact, callback): + from contact import Contact + contact.contact_user_id = None + self._LogUpdate(contact) + if Version._mutate_items: + yield gen.Task(contact.Update, client) + + callback(contact) + +class UploadContactsSupport(Version): + """Convert contact records to support upload_contacts schema change. + - Populate new fields: + * timestamp: Generate new timestamp for each record. + * contact_source: 'gm' from 'Email:' identity or 'fb' from 'FacebookGraph:' identity. + * identities: Create with one value from existing identity column. + * contact_id: Derived from base64/hash of other columns plus contact_source. + * identities_properties: Create with one value from existing identity column and None for description. + - Update existing field: + * sort_key: Now, derived from timestamp and contact_id columns. + """ + @gen.engine + def Transform(self, client, contact, callback): + from viewfinder.backend.db.contact import Contact + from viewfinder.backend.db.identity import Identity + contact_dict = contact._asdict() + # During this upgrade assume that any email identities came from GMail and any facebook identities came from + # Facebook. At this time, there shouldn't be any identities that start with anything else. + assert contact.identity.startswith('Email:') or contact.identity.startswith('FacebookGraph:'), contact + contact_dict['contact_source'] = Contact.GMAIL if contact.identity.startswith('Email:') else Contact.FACEBOOK + contact_dict['identities_properties'] = [(Identity.Canonicalize(contact.identity), None)] + contact_dict['timestamp'] = util.GetCurrentTimestamp() + # Let Contact.CreateFromKeywords calculate a new sort_key. + contact_dict.pop('sort_key') + # Let Contact.CreateFromKeywords determine value for identities column. + contact_dict.pop('identities') + # Contact.CreateFromKeywords() will calculate sort_key, contact_id, and identities columns. + new_contact = Contact.CreateFromKeywords(**contact_dict) + + self._LogUpdate(new_contact) + + if Version._mutate_items: + yield gen.Task(new_contact.Update, client) + yield gen.Task(contact.Delete, client) + + callback(new_contact) + +class RenameUserPostRemoved(Version): + """Rename the USER_POST "REMOVED" label to be "HIDDEN".""" + @gen.engine + def Transform(self, client, user_post, callback): + from user_post import UserPost + if UserPost.REMOVED in user_post.labels: + labels = user_post.labels.combine() + labels.remove(UserPost.REMOVED) + labels.add(UserPost.HIDDEN) + user_post.labels = labels + self._LogUpdate(user_post) + if Version._mutate_items: + yield gen.Task(user_post.Update, client) + + callback(user_post) + +class AddRemovedToPost(Version): + """Add REMOVED to every UNSHARED post.""" + @gen.engine + def Transform(self, client, post, callback): + from post import Post + labels = post.labels.combine() + if Post.UNSHARED in labels: + labels.add(Post.REMOVED) + post.labels = labels + self._LogUpdate(post) + if Version._mutate_items: + yield gen.Task(post.Update, client) + + callback(post) + +class RemoveHiddenPosts(Version): + """Add REMOVED label to posts that are hidden in default viewpoints, and then delete the + user post. + """ + @gen.engine + def Transform(self, client, user_post, callback): + from episode import Episode + from post import Post + from user_post import UserPost + from viewpoint import Viewpoint + + if UserPost.REMOVED in user_post.labels: + episode_id, photo_id = Post.DeconstructPostId(user_post.post_id) + episode = yield gen.Task(Episode.Query, client, episode_id, None) + viewpoint = yield gen.Task(Viewpoint.Query, client, episode.viewpoint_id, None) + if viewpoint.IsDefault(): + post = yield gen.Task(Post.Query, client, episode_id, photo_id, None) + post.labels.add(Post.REMOVED) + + logging.info('Adding REMOVED label to POST: %s (%s, %s)', user_post.post_id, episode_id, photo_id) + if Version._mutate_items: + yield gen.Task(post.Update, client) + yield gen.Task(user_post.Delete, client) + + callback(user_post) + +class ReactivateAlertUser(Version): + """Set Device.alert_user_id for devices that have a push token and belong to a non-terminated user.""" + @gen.engine + def Transform(self, client, device, callback): + from device import Device + from user import User + + if device.push_token is not None and device.alert_user_id is None: + user = yield gen.Task(User.Query, client, device.user_id, None) + if not user.IsTerminated(): + device.alert_user_id = device.user_id + if Version._mutate_items: + yield gen.Task(device.Update, client) + + callback(device) + +class RemoveContactIdentity(Version): + """Remove the identity field from contact records.""" + @gen.engine + def Transform(self, client, contact, callback): + contact.identity = None + self._LogUpdate(contact) + if Version._mutate_items: + yield gen.Task(contact.Update, client) + + callback(contact) + +class RepairWelcomeConvos(Version): + """Repair the welcome conversations by updating the cover_photo episode id to be an episode + in the viewpoint. + """ + @gen.engine + def Transform(self, client, viewpoint, callback): + from viewpoint import Viewpoint + from viewfinder.backend.www import system_users + + if viewpoint.type == Viewpoint.SYSTEM: + episodes, _ = yield gen.Task(Viewpoint.QueryEpisodes, client, viewpoint.viewpoint_id) + for episode in episodes: + if episode.parent_ep_id == 'egAZn7AjQ-F7': + assert viewpoint.cover_photo['episode_id'] == episode.parent_ep_id or episode.episode_id, episode + assert viewpoint.cover_photo['photo_id'] == 'pgAZn7AjQ-FB', viewpoint.cover_photo + + viewpoint.cover_photo = {'episode_id': episode.episode_id, + 'photo_id': 'pgAZn7AjQ-FB'} + + self._LogUpdate(viewpoint) + if Version._mutate_items: + yield gen.Task(viewpoint.Update, client) + + callback(viewpoint) + +class RepairWelcomeConvos2(Version): + """Repair corrupt welcome conversations by removing all photos.""" + @gen.engine + def Transform(self, client, viewpoint, callback): + from viewpoint import Viewpoint + from viewfinder.backend.www import system_users + + if viewpoint.type == Viewpoint.SYSTEM: + episodes, _ = yield gen.Task(Viewpoint.QueryEpisodes, client, viewpoint.viewpoint_id) + if len(episodes) == 0: + # Remove reference to cover photo. + viewpoint.cover_photo = None + self._LogUpdate(viewpoint) + if Version._mutate_items: + yield gen.Task(viewpoint.Update, client) + + # Delete share_existing activities. + activities, _ = yield gen.Task(Viewpoint.QueryActivities, client, viewpoint.viewpoint_id) + for activity in activities: + if activity.name == 'share_existing': + logging.info('removing activity %s' % activity.activity_id) + if Version._mutate_items: + yield gen.Task(activity.Delete, client) + + # Remove comment reference to photo asset. + comments, _ = yield gen.Task(Viewpoint.QueryComments, client, viewpoint.viewpoint_id) + for comment in comments: + if comment.asset_id is not None: + comment.asset_id = None + self._LogUpdate(comment) + if Version._mutate_items: + yield gen.Task(comment.Update, client) + + callback(viewpoint) + + +# Append new version migrations here. +# MAINTAIN THE ORDERING! +# ONLY DELETE OLDEST CLASSES IN ORDER! +# IF YOU DELETE, MAKE SURE TO UPDATE Version._DELETED_MIGRATOR_COUNT. +TEST_VERSION = TestVersion() +TEST_VERSION2 = TestVersion2() +ADD_VIEWPOINT_SEQ = AddViewpointSeq() +ADD_ACTIVITY_SEQ = AddActivitySeq() +UPDATE_ACTIVITY_SHARE = UpdateActivityShare() +CREATE_MD5_HASHES = CreateMD5Hashes() +DISAMBIGUATE_ACTIVITY_IDS = DisambiguateActivityIds() +MIGRATE_MORE_SHARES = MigrateMoreShares() +ADD_USER_TYPE = AddUserType() +DISAMBIGUATE_ACTIVITY_IDS_2 = DisambiguateActivityIds2() +COPY_UPDATE_SEQ = CopyUpdateSeq() +ADD_USER_SIGNING_KEY = AddUserSigningKey() +UPDATE_USER_TYPE = UpdateUserType() +ADD_FOLLOWED = AddFollowed() +UPDATE_DEVICES = UpdateDevices() +FILL_FILE_SIZES = FillFileSizes() +REPAIR_FACEBOOK_CONTACTS = RepairFacebookContacts() +ADD_ACCOUNT_SETTINGS = AddAccountSettings() +SPLIT_USER_NAMES = SplitUserNames() +EXTRACT_ASSET_KEYS = ExtractAssetKeys() +MY_ONLY_FRIEND = MyOnlyFriend() +ERASE_FRIEND_NAMES = EraseFriendNames() +MOVE_DEVICE_PHOTO = MoveDevicePhoto() +SET_COVER_PHOTO = SetCoverPhoto() +REMOVE_ASSET_URLS = RemoveAssetUrls() +REMOVE_CONTACT_USER_ID = RemoveContactUserId() +UPLOAD_CONTACTS_SUPPORT = UploadContactsSupport() +RENAME_USER_POST_REMOVED = RenameUserPostRemoved() +ADD_REMOVED_TO_POST = AddRemovedToPost() +REMOVE_HIDDEN_POSTS = RemoveHiddenPosts() +REACTIVATE_ALERT_USER = ReactivateAlertUser() +REMOVE_CONTACT_IDENTITY = RemoveContactIdentity() +REPAIR_WELCOME_CONVOS = RepairWelcomeConvos() +REPAIR_WELCOME_CONVOS_2 = RepairWelcomeConvos2() + +# TODO(spencer): should transform all photo placemarks by simply +# loading them and then saving them. That will remove all of the +# incorrectly-padded base64-encoded values. +# +# Also, should consider re-indexing all full-text-search indexed +# columns for the same reason. diff --git a/backend/db/vf_schema.py b/backend/db/vf_schema.py new file mode 100644 index 0000000..0ba8841 --- /dev/null +++ b/backend/db/vf_schema.py @@ -0,0 +1,728 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""The Viewfinder schema definition. + +The schema contains a set of tables. Each table is described by name, +key, a set of columns, and a list of versions. + +The table name is the name used to access the database. The table key +is used to segment index terms by table. It, combined with the column +key, forms the prefix of each index term generated when a column value +is indexed. +""" + +__authors__ = ['spencer@emailscrubbed.com (Spencer Kimball)', + 'andy@emailscrubbed.com (Andy Kimball)'] + +from schema import Schema, Table, IndexedTable, IndexTable, Column, HashKeyColumn, RangeKeyColumn, SetColumn, JSONColumn, LatLngColumn, PlacemarkColumn, CryptColumn +from indexers import Indexer, SecondaryIndexer, FullTextIndexer, EmailIndexer, LocationIndexer + +ACCOUNTING = 'Accounting' +ACTIVITY = 'Activity' +ADMIN_PERMISSIONS = 'AdminPermissions' +ANALYTICS = "Analytics" +COMMENT = 'Comment' +CONTACT = 'Contact' +DEVICE = 'Device' +EPISODE = 'Episode' +FOLLOWED = 'Followed' +FOLLOWER = 'Follower' +FRIEND = 'Friend' +GUESS = 'Guess' +HEALTH_REPORT = 'HealthReport' +ID_ALLOCATOR = 'IdAllocator' +IDENTITY = 'Identity' +LOCK = 'Lock' +METRIC = 'Metric' +NOTIFICATION = 'Notification' +OPERATION = 'Operation' +PHOTO = 'Photo' +POST = 'Post' +SETTINGS = 'Settings' +SHORT_URL = 'ShortURL' +SUBSCRIPTION = 'Subscription' +USER = 'User' +USER_PHOTO = 'UserPhoto' +USER_POST = 'UserPost' +VIEWPOINT = 'Viewpoint' +INDEX = 'Index' + +TEST_RENAME = 'TestRename' + +SCHEMA = Schema([ + # The accounting table stores aggregated usage stats. + # The hash and sort keys are strings consisting of 'prefix:' + # + # Accounting categories: + # - Per viewpoint: hash_key='vs:' + # Aggregate sizes/counts per viewpoint, keyed by the viewpoint + # id. Sort keys fall into three categories: + # - owned by: 'ow:' only found in default viewpoint. + # - shared by: 'sb:' in shared viewpoint, sum of all photos + # in episodes owned by 'user_id' + # - visible to: 'vt' in shared viewpoint, sum of all photos. not keyed + # by user. a given user's "shared with" stats will be 'vt - sb:', + # but we do not want to keep per-user shared-by stats. + # - Per user: hash_key='us:' + # Aggregate sizes/counts per user, keyed by user id. Sort keys are: + # - owned by: 'ow' sum of all photos in default viewpoint + # - shared by: 'sb' sum of all photos in shared viewpoints and episodes owned by this user + # - visible to: 'vt' sum of all photos in shared viewpoint (includes 'sb'). to get the + # real count of photos shared with this user but not shared by him, compute 'vt - sb' + # + # 'op_ids' holds a list of previously-applied operation IDs. This is an attempt to + # make increments idempotent with replays. The list is a comma-separated string of + # operation ids (sometimes suffixed with a viewpoint ID), in the order in which they were + # applied. We keep a maximum of Accounting._MAX_APPLIED_OP_IDS. + # + # Currently, all columns are used by each accounting category. + Table(ACCOUNTING, 'at', read_units=100, write_units=10, + columns=[HashKeyColumn('hash_key', 'hk', 'S'), + RangeKeyColumn('sort_key', 'sk', 'S'), + Column('num_photos', 'np', 'N'), + Column('tn_size', 'ts', 'N'), + Column('med_size', 'ms', 'N'), + Column('full_size', 'fs', 'N'), + Column('orig_size', 'os', 'N'), + Column('op_ids', 'oi', 'S')]), + + # Activities are associated with a viewpoint and contain a record of + # all high-level operations which have modified the structure of the + # viewpoint in some way. For more details, see activity.py. The + # activity_id attribute is a composite of information gleaned from + # current operation: (reverse timestamp, user_id, op_id). The content + # of the activity is a JSON-encoded ACTIVITY structure, as defined in + # json_schema.py. 'update_seq' is set to the value of the viewpoint's + # 'update_seq' attribute after it was incremented during creation of + # the activity. + Table(ACTIVITY, 'ac', read_units=100, write_units=10, + columns=[HashKeyColumn('viewpoint_id', 'vi', 'S'), + RangeKeyColumn('activity_id', 'ai', 'S'), + Column('user_id', 'ui', 'N', read_only=True), + Column('timestamp', 'ti', 'N', read_only=True), + Column('update_seq', 'us', 'N'), + Column('name', 'na', 'S', read_only=True), + Column('json', 'js', 'S', read_only=True)]), + + # Admin table. This table lists all users with access to admin and support functions. + # Entries are created by the otp script, with 'rights' being a set of roles (eg: 'root' or 'support'). + # Admin users are not currently linked to viewfinder users. + Table(ADMIN_PERMISSIONS, 'ad', read_units=10, write_units=10, + columns=[HashKeyColumn('username', 'un', 'S'), + Column('rights', 'ri', 'SS')]), + + # Timestamped information for various entities. The entity hash key should be of the form: :. + # eg: us:112 (for user with ID 112). + # sort_key: base64 hex encoded timestamp + type + # Type is a string representing the type of analytics entry. See db/analytics.py for details. + # Payload is an optional payload attached to the entry. Its format depends on the type of entry. + Table(ANALYTICS, 'an', read_units=10, write_units=10, + columns=[HashKeyColumn('entity', 'et', 'S'), + RangeKeyColumn('sort_key', 'sk', 'S'), + Column('timestamp', 'ti', 'N'), + Column('type', 'tp', 'S'), + Column('payload', 'pl', 'S')]), + + # Key is composite of (viewpoint_id, comment_id), which sorts all + # comments by ascending timestamp within each viewpoint. 'user_id' + # is the user that created the comment. At this time, 'asset_id' + # can be: + # 1. Absent: The comment is not linked to any other asset. + # 2. Comment id: The comment is a response to another comment. + # 3. Photo id: The comment is a comment on a photo. + # + # 'timestamp' records the time that the comment was originally + # created. 'message' is the actual comment text. + IndexedTable(COMMENT, 'cm', read_units=200, write_units=20, + columns=[HashKeyColumn('viewpoint_id', 'vi', 'S'), + RangeKeyColumn('comment_id', 'ci', 'S'), + Column('user_id', 'ui', 'N', read_only=True), + Column('asset_id', 'ai', 'S', read_only=True), + Column('timestamp', 'ti', 'N'), + Column('message', 'me', 'S')]), + + # Key is composite of (user_id, sort_key) + # sort_key: base64 hex encoded timestamp + contact_id + # contact_id: contact_source + ':' + hash (base64 encoded) of CONTACT data: name, given_name, family_name, + # rank, and identities_properties columns. + # contact_source: 'fb', 'gm', 'ip', or 'm' (for, respectively, Facebook, GMail, iPhone, and Manual sources) + # timestamp column should always match the timestamp encoded prefix of the sort_key. + # identities: set of canonicalized identity strings: Email:, Phone:, Facebook: + # These reference identities in the IDENTITY table. This column exists so that contacts can be queried by + # identity. Note: duplicates info that's contained in the identities_properties column. + # identities_properties: json formatted list of identities each with an optional label such as 'mobile', 'work', + # etc... This list preserves the order in which the identities were upload by (or fetched from) a + # contact source. These identities may not be in canonicalized form, but it must be possible to canonicalize + # them. + # labels: 'removed' indicates that the contact is in a removed state. This surfaces the removed state of + # contacts to clients through invalidation notifications. These contacts will be filtered out for down-level + # client queries. + IndexedTable(CONTACT, 'co', read_units=50, write_units=120, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('sort_key', 'sk', 'S'), + Column('timestamp', 'ti', 'N'), + Column('contact_id', 'ci', 'S', SecondaryIndexer(), read_only=True), + Column('contact_source', 'cs', 'S', read_only=True), + SetColumn('labels', 'lb', 'SS'), + SetColumn('identities', 'ids', 'SS', SecondaryIndexer(), read_only=True), + Column('name', 'na', 'S', read_only=True), + Column('given_name', 'gn', 'S', read_only=True), + Column('family_name', 'fn', 'S', read_only=True), + Column('rank', 'ra', 'N', read_only=True), + JSONColumn('identities_properties', 'ip', read_only=True)]), + + # Device information. Key is a composite of user id and a 32-bit + # integer device id (allocated via the id-allocation table). Each + # device is a source of photos. The device id comprises the first + # 32 bits of the photo id. The last 32 bits are sequentially + # allocated by the device (in the case of mobile), or via an + # atomic increment of 'id_seq' (in the case of the web). + # + # 'last_access' and 'push_token' are set on device registration + # and each time the application is launched (in the case of the + # mobile app). 'push_token' is indexed to allow device lookups in + # response to feedback from provider push-notification services. + # 'alert_user_id' is a sparse column index, used to quickly find + # all devices for a user that need to be alerted. + # + # Device ID of 0 is reserved to mean local to an individual device. + # + # Example Apple push token: "apns:oYJrenW5JsH42r1eevgq3HhC6bhXL3OP0SqHkOeo/58=" + IndexedTable(DEVICE, 'de', read_units=25, write_units=5, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('device_id', 'di', 'N', SecondaryIndexer()), + Column('timestamp', 'ti', 'N'), + Column('name', 'na', 'S'), + Column('version', 've', 'S'), + Column('platform', 'pl', 'S'), + Column('os', 'os', 'S'), + Column('last_access', 'la', 'N'), + Column('alert_user_id', 'aui', 'N', SecondaryIndexer()), + Column('push_token', 'pt', 'S', SecondaryIndexer()), + Column('language', 'lg', 'S'), + Column('country', 'co', 'S')]), + + # Key is episode-id. Episodes are indexed for full-text search on + # episode title and description, and lookup of all episodes for a user. + # Due to a rename, the Episode table is called Event in the database. + IndexedTable(EPISODE, 'ev', read_units=200, write_units=10, name_in_db="Event", + columns=[HashKeyColumn('episode_id', 'ei', 'S'), + Column('parent_ep_id', 'pa', 'S', SecondaryIndexer(), read_only=True), + Column('user_id', 'ui', 'N', SecondaryIndexer(), read_only=True), + Column('viewpoint_id', 'vi', 'S', SecondaryIndexer(), read_only=True), + Column('publish_timestamp', 'pu', 'N'), + Column('timestamp', 'cr', 'N'), + Column('title', 'ti', 'S'), + Column('description', 'de', 'S'), + LatLngColumn('location', 'lo'), + PlacemarkColumn('placemark', 'pl')]), + + # Sorts all viewpoints followed by a user in order of the date of + # on which the last activity was added. Viewpoints updated on the + # same day are in undefined order. Sort is in descending order, with + # viewpoints most recently updated coming first. The query_followed + # method returns results in this ordering. Note that paging may result + # in missed followed records, as updates to a viewpoint may cause the + # corresponding record to "jump ahead" in time past the current paging + # bookmark. 'date_updated' is a timestamp truncated to a day boundary. + # 'sort_key' is a concatenation of the 'date_updated' field and the + # viewpoint id. + IndexedTable(FOLLOWED, 'fd', read_units=200, write_units=10, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('sort_key', 'sk', 'S'), + Column('date_updated', 'du', 'N'), + Column('viewpoint_id', 'vi', 'S', read_only=True)]), + + # Key is a composite of (user-id, viewpoint-id). The 'labels' set + # specifies the features of the relation between the user and + # viewpoint: ('admin', 'contribute'). 'adding_user_id' contains the id + # of the user who added this follower, and 'timestamp' the time at which + # the follower was added. 'viewed_seq' is the sequence number of the last + # viewpoint update that has been 'read' by this follower. The last + # viewpoint update is tracked by the 'update_seq' attribute on Viewpoint. + IndexedTable(FOLLOWER, 'fo', read_units=400, write_units=10, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('viewpoint_id', 'vi', 'S', SecondaryIndexer()), + Column('timestamp', 'ti', 'N'), + Column('adding_user_id', 'aui', 'N'), + SetColumn('labels', 'la', 'SS'), + Column('viewed_seq', 'vs', 'N')]), + + # Key is composite of user-id / friend-id. "colocated_shares" and + # "total_shares" are decaying stats that track the number of photo + # opportunities where sharing occurred. 'last_colocated' and + # 'last_share' are timestamps for computing decay. Friend status is + # one of {friend,blocked,muted}. + Table(FRIEND, 'fr', read_units=50, write_units=10, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('friend_id', 'fi', 'N'), + Column('name', 'na', 'S'), + Column('nickname', 'nn', 'S'), + Column('colocated_shares', 'cs', 'N'), + Column('last_colocated', 'lc', 'N'), + Column('total_shares', 'ts', 'N'), + Column('last_share', 'ls', 'N'), + Column('status', 'st', 'S')]), + + # Tracks the number of incorrect attempts that have been made to guess some + # secret, such as a password or an access code. 'guess_id' is of the form + # :, where is one of these: + # + # url: - Limits number of attempts that can be made to guess a + # valid ShortURL within any particular 24-hour period. + # + # pw: - Limits number of attempts that can be made to guess a + # particular user's password within any particular 24-hour + # period. + # + # em: - Limits number of attempts that can be made to guess + # access tokens e-mailed to a particular user within any + # particular 24-hour period. + # + # ph: - Limits number of attempts that can be made to guess + # access tokens sent in SMS messages to a user within any + # particular 24-hour period. + # + # The 'guesses' field tracks the number of incorrect guesses that have been + # made so far. The 'expires' field stores the time at which the guesses count + # can be reset to 0. + Table(GUESS, 'gu', read_units=50, write_units=10, + columns=[HashKeyColumn('guess_id', 'gi', 'S'), + Column('expires', 'ex', 'N'), + Column('guesses', 'gu', 'N')]), + + # Key is a composite of (group_key, timestamp), where group_key is the + # same key used to collect machine metrics in the metrics table. The + # intention is that for each metrics group_key, a single health report + # will be generated summarizing problems across all machines in that group. + # + # Alerts and Warnings are string sets which describe any problems detected + # from the metrics information. If no problems are detected, this record + # will be sparse. + Table(HEALTH_REPORT, 'hr', read_units=10, write_units=5, + columns=[HashKeyColumn('group_key', 'gk', 'S'), + RangeKeyColumn('timestamp', 'ts', 'N'), + SetColumn('alerts', 'as', 'SS'), + SetColumn('warnings', 'ws', 'SS')]), + + # Key is ID type (e.g. op-id, photo-id, user-id, episode-id). + Table(ID_ALLOCATOR, 'ia', read_units=10, write_units=10, + columns=[HashKeyColumn('id_type', 'it', 'S'), + Column('next_id', 'ni', 'N')]), + + # Key is identity. User-id is indexed to provide quick queries for the + # list of identities associated with a viewfinder account. The token + # allows access to external resources associated with the identity. + # 'last_fetch' specifies the last time that the contacts were + # fetched for this identity. 'authority' is one of ('Facebook', 'Google' + # 'Viewfinder', etc.) and identifies the trusted authentication authority. + # + # The complete set of attributes (if any) returned when an + # identity was authenticated is stored as a json-encoded dict in + # 'json_attrs'. Some of these may be taken to populate the + # demographic and informational attributes of the User table. + # + # The 'access_token' and 'refresh_token' fields store any tokens used to + # access the authority, with 'expires' tracking the lifetime of the + # token. + # + # The 'auth_throttle' field limits the number of auth email/sms messages + # that can be sent within a certain period of time. + IndexedTable(IDENTITY, 'id', read_units=50, write_units=10, + columns=[HashKeyColumn('key', 'ke', 'S'), + Column('user_id', 'ui', 'N', SecondaryIndexer()), + JSONColumn('json_attrs', 'ja'), + Column('last_fetch', 'lf', 'N'), + Column('authority', 'au', 'S'), + Column('access_token', 'at', 'S'), + Column('refresh_token', 'rt', 'S'), + Column('expires', 'ex', 'N'), + JSONColumn('auth_throttle', 'th'), + + # TODO(Andy): Remove these attributes, as they are now deprecated. + Column('access_code', 'ac', 'S', SecondaryIndexer()), + Column('expire_code', 'xc', 'N'), + Column('token_guesses', 'tg', 'N'), + Column('token_guesses_time', 'gt', 'N')]), + + # A lock is acquired in order to control concurrent access to + # a resource. The 'lock_id' is a composite of the type of the + # resource and its unique id. The 'owner_id' is a string that + # uniquely identifies the holder of the lock. 'resource_data' + # is resource-specific information that is provided by the + # owner and stored with the lock. The 'expiration' is the time + # (UTC) at which the lock is assumed to have been abandoned by + # the owner and can be taken over by another owner. + # + # 'acquire_failures' tracks the number of times other agents + # tried to acquire the lock while it was held. + Table(LOCK, 'lo', read_units=50, write_units=10, + columns=[HashKeyColumn('lock_id', 'li', 'S'), + Column('owner_id', 'oi', 'S'), + Column('expiration', 'ex', 'N'), + Column('acquire_failures', 'af', 'N'), + Column('resource_data', 'rd', 'S')]), + + # Metrics represent a timestamped payload of performance metrics + # from a single machine running viewfinder. The metrics key is a + # composite of (group_key, sort_key). The payload column is a serialized + # dictionary describing the performance metrics that were captured from + # the machine. + # + # The group_key for a metric is intended to organize metrics by the way + # they are queried. For instance, a group key might contain all + # metrics for all machines in an EC2 region, or a more specific division + # than that. + # + # The sort_key is a composite of the timestamp and machine id - the + # intention is that records will be queried by timestamp, while machine_id + # is simply included in the key to differentiate records with the same + # timestamp from different machines. + IndexedTable(METRIC, 'mt', read_units=50, write_units=10, + columns=[HashKeyColumn('group_key', 'gk', 'S'), + RangeKeyColumn('sort_key', 'sk', 'S'), + Column('machine_id', 'mi', 'S', SecondaryIndexer()), + Column('timestamp', 'ts', 'N'), + Column('payload', 'p', 'S')]), + + # Notifications are messages to deliver to devices hosting the + # viewfinder client, whether mobile, desktop, web application or + # otherwise. Key is a composite of (user-id and allocated + # notification id--taken from user's uu_id sequence). Other + # fields record the name, id, and timestamp of the operation that + # resulted in the notification, as well as the user and device + # that started it. The badge attribute records the value of the + # "push badge" on client devices at the time that notification + # was recorded. The invalidate attribute is a JSON-encoded + # INVALIDATE structure, as defined in json_schema.py. + Table(NOTIFICATION, 'no', read_units=50, write_units=10, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('notification_id', 'ni', 'N'), + Column('name', 'na', 'S'), + Column('timestamp', 'ti', 'N'), + Column('sender_id', 'si', 'N'), + Column('sender_device_id', 'sd', 'N'), + Column('badge', 'ba', 'N'), + Column('invalidate', 'in', 'S'), + Column('op_id', 'oi', 'S'), + Column('viewpoint_id', 'vi', 'S'), + Column('update_seq', 'us', 'N'), + Column('viewed_seq', 'vs', 'N'), + Column('activity_id', 'ai', 'S')]), + + # Operations are write-ahead logs of mutating server + # requests. These requests need to be persisted so that they can + # be retried on server failure. They often involve multiple + # queries / updates to different database tables and/or rows, so a + # partially completed operation could leave the database in an + # inconsistent state. Each operation must be idempotent, as + # failing servers may cause retries. The actual operation is + # stored JSON-encoded in 'json'. This is often the original HTTP + # request, though in some cases, the JSON from the HTTP request + # is augmented with additional information, such as pre-allocated + # photo, user or device IDs. + # + # 'quarantine' indicates that if the operation fails, it + # should not prevent further operations for the same user from + # processing. + # + # 'checkpoint' stores progress information with the operation. If the + # operation is restarted, it can use this information to skip over + # steps it's already completed. The progress information is operation- + # specific and is not used in any way by the operation framework itself. + # + # 'triggered_failpoints' is used for testing operation idempotency. It + # contains the set of failpoints which have already been triggered for + # this operation and need not be triggered again. + Table(OPERATION, 'op', read_units=50, write_units=50, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('operation_id', 'oi', 'S'), + Column('device_id', 'di', 'N'), + Column('method', 'me', 'S'), + Column('json', 'js', 'S'), + Column('timestamp', 'ti', 'N'), + Column('attempts', 'at', 'N'), + Column('backoff', 'bo', 'N'), + Column('first_failure', 'ff', 'S'), + Column('last_failure', 'lf', 'S'), + Column('quarantine', 'sf', 'N'), + JSONColumn('checkpoint', 'cp'), + JSONColumn('triggered_failpoints', 'fa')]), + + # Key is photo-id. Photo id is composed of 32 bits of time in the + # high 32 bits, then 32 bits of device id, then 32 bits of + # monotonic photo id, unique to the device. The full 96 bits are + # base-64 hex encoded into 128 bits. Photos can have a parent + # photo-id, which refers back to an original photo if this is a + # copy. Copies are made when filters are applied to photos. The + # client_data string is a JSON-encoded dict of opaque + # client-supplied key-value pairs. + # + # The 'share_seq_no' attribute is incremented every time the shares + # for a photo are modified. It provides for efficient queries from + # clients meant to determine the list of friends with viewing + # privileges + # + # Sizes for tn, med, full, orig are file sizes in bytes for thumnail, + # medium, full and original images respectively. + # + # The 'new_assets' attribute is temporary and there to support rename + # of image asset files from underscore to period suffixes. It contains + # the value 'copied' if the asset files have been duplicated and + # 'deleted' if the original asset files have been verified as copied + # and removed. + # TODO(spencer): remove this once we have completely migrated the photo + # data. + # + # 'client_data' is deprecated; use USER_PHOTO instead. + IndexedTable(PHOTO, 'ph', read_units=400, write_units=25, + columns=[HashKeyColumn('photo_id', 'pi', 'S'), + Column('parent_id', 'pa', 'S', SecondaryIndexer(), read_only=True), + Column('episode_id', 'ei', 'S', read_only=True), + Column('user_id', 'ui', 'N', read_only=True), + Column('aspect_ratio', 'ar', 'N'), + Column('content_type', 'ct', 'S', read_only=True), + Column('timestamp', 'ti', 'N'), + Column('tn_md5', 'tm', 'S'), + Column('med_md5', 'mm', 'S'), + Column('orig_md5', 'om', 'S', SecondaryIndexer()), + Column('full_md5', 'fm', 'S', SecondaryIndexer()), + Column('tn_size', 'ts', 'N'), + Column('med_size', 'ms', 'N'), + Column('full_size', 'fs', 'N'), + Column('orig_size', 'os', 'N'), + LatLngColumn('location', 'lo'), + PlacemarkColumn('placemark', 'pl'), + Column('caption', 'ca', 'S', + FullTextIndexer(metaphone=Indexer.Option.YES)), + Column('link', 'li', 'S'), + Column('thumbnail_data', 'da', 'S'), + Column('share_seq', 'ss', 'N'), + JSONColumn('client_data', 'cd'), # deprecated + Column('new_assets', 'na', 'S')]), + + # Key is composite of (episode-id, photo_id). When photos are + # posted/reposted to episodes, a post relation is created. This + # allows the same photo to be included in many episodes. The + # 'labels' attribute associates a set of properties with the + # post. + IndexedTable(POST, 'po', read_units=200, write_units=25, + columns=[HashKeyColumn('episode_id', 'ei', 'S'), + RangeKeyColumn('photo_id', 'sk', 'S'), + SetColumn('labels', 'la', 'SS')]), + + # Key is composite of (settings_id, group_name). 'settings_id' is the + # id of the entity to which the settings apply. For example, user account + # settings have ids like 'us:'. 'group_name' can be used if + # a particular entity has large numbers of settings that need to be + # sub-grouped. + # + # All other columns are a union of all columns defined by all the groups + # stored in the table. The Settings class has support for only exposing + # columns that apply to a particular group, in order to avoid accidental + # use of a column belonging to another settings group. + Table(SETTINGS, 'se', read_units=100, write_units=10, + columns=[HashKeyColumn('settings_id', 'si', 'S'), + RangeKeyColumn('group_name', 'gn', 'S'), + + # User account group settings. + Column('user_id', 'ui', 'N'), + Column('email_alerts', 'ea', 'S'), + Column('sms_alerts', 'sa', 'S'), + Column('push_alerts', 'pa', 'S'), + Column('marketing', 'mk', 'S'), + Column('sms_count', 'sc', 'N'), + SetColumn('storage_options', 'so', 'SS')]), + + # Key is composite of (group_id, random_key). 'group_id' partitions the URL + # space into groups, so that URL's generated for one group have no overlap + # with those for another group. The group id will be appended as the URL + # path, so it may contain '/' characters, and should be URL safe. The + # 'timestamp' column tracks the time at which the ShortURL was created. + # + # The 'json' column contains arbitrary named arguments that are associated + # with the short URL and are pased to the request handler when the short + # URL is used. The 'expires' field bounds the time during which the URL + # can be used. + Table(SHORT_URL, 'su', read_units=25, write_units=5, + columns=[HashKeyColumn('group_id', 'gi', 'S'), + RangeKeyColumn('random_key', 'rk', 'S'), + Column('timestamp', 'ti', 'N'), + Column('expires', 'ex', 'N'), + JSONColumn('json', 'js')]), + + # The subscription table contains a user's current + # subscription(s). A subscription is any time-limited + # modification to a user's privileges, such as increased storage + # quota. + # + # This table contains a log of all transactions that have affected + # a user's subscriptions. In most cases only the most recent + # transaction for a given subscription_id is relevant - it is the + # most recent renewal. + # + # "product_type" is the type of subscription, such as "storage". + # Quantity is a interpreted based on the product_type; for the + # "storage" product it is a number of GB. "payment_type" + # indicates how the subscription was paid for (e.g. "itunes" or + # "referral_bonus"). The contents of "extra_info" and + # "renewal_data" depend on the payment type. "extra_info" is a + # dict of additional information related to the transaction, and + # "renewal_data" is an opaque blob that is used to renew a subscription + # when it expires. + Table(SUBSCRIPTION, 'su', read_units=10, write_units=5, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('transaction_id', 'tr', 'S'), + Column('subscription_id', 'su', 'S', read_only=True), + # timestamps should be read-only too, once we fix + # problems with read-only floats. + Column('timestamp', 'ti', 'N'), + Column('expiration_ts', 'ex', 'N'), + Column('product_type', 'pr', 'S', read_only=True), + Column('quantity', 'qu', 'N'), + Column('payment_type', 'pt', 'S', read_only=True), + JSONColumn('extra_info', 'ei'), + Column('renewal_data', 'pd', 'S', read_only=True)]), + + # Key is user id. 'webapp_dev_id' is assigned on creation, and + # serves as a unique ID with which to formulate asset IDs in + # conjunction with the 'asset_id_seq' attribute. This provides a + # monotonically increasing sequence of episode/viewpoint/photo ids + # for uploads via the web application. The 'uu_id_seq' provides a + # similar increasing sequence of user update sequence numbers for + # a user. + # + # Facebook email is kept separately in an effort to maximize + # deliverability of Viewfinder invitations to Facebook contacts. + # The from: header of those emails must be from the email address + # registered for the Facebook user if incoming to + # @facebook.com. + # + # 'last_notification' is the most recent notification id which has + # been queried by any of the user's devices. This is the watermark + # used to supply the badge for push notifications. 'badge' is set + # appropriately in response to notifications generated by account + # activity. + # + # The 'merged_with' column specifies the sink user account with + # which this user was merged. If 'merged_with' is set, this user + # account is invalid and should not be used. If at all possible, + # the request intended for this user should be re-routed to the + # 'merged_with' user. + # + # The 'signing_key' column is a Keyczar signing keyset used when + # it is desirable to sign a payload with a key that is specific to + # one particular user. The contents of the column are encrypted + # with the service-wide db crypt keyset. + # + # The 'pwd_hash' and 'salt' columns are used to securely generate + # and store an iterative SHA1 hash of the user's password + salt. + # + # For user index, range key column is a string-version of user ID. + IndexedTable(USER, 'us', read_units=50, write_units=50, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + Column('private_vp_id', 'pvi', 'S'), + Column('webapp_dev_id', 'wdi', 'N'), + Column('asset_id_seq', 'ais', 'N'), + Column('uu_id_seq', 'uis', 'N'), + Column('given_name', 'fi', 'S', FullTextIndexer()), + Column('family_name', 'la', 'S', FullTextIndexer()), + Column('name', 'na', 'S', FullTextIndexer()), + Column('email', 'em', 'S', EmailIndexer()), + Column('facebook_email', 'fe', 'S'), + LatLngColumn('location', 'lo', LocationIndexer()), + Column('gender', 'ge', 'S'), + Column('locale', 'lc', 'S'), + Column('link', 'li', 'S'), + Column('phone', 'ph', 'S'), + Column('picture', 'pi', 'S'), + Column('timezone', 'ti', 'N'), + Column('last_notification', 'ln', 'N'), + Column('badge', 'ba', 'N'), + Column('merged_with', 'mw', 'N'), + SetColumn('labels', 'lb', 'SS'), + CryptColumn('signing_key', 'sk'), + CryptColumn('pwd_hash', 'pwd'), + CryptColumn('salt', 'slt'), + + # Deprecated (to be removed). + Column('beta_status', 'bs', 'S')]), + + # The USER_PHOTO is associated with a PHOTO object, and + # represents user-specific information about the photo. + # Specifically, this includes mappings between the photo and a + # device's native asset library. Normally only the user/device + # who originated the photo will have a USER_PHOTO entry for it, + # but it is possible for other users to create USER_PHOTOS if + # they export a photo to their camera roll. + IndexedTable(USER_PHOTO, 'up', read_units=400, write_units=10, + columns=[HashKeyColumn('user_id', 'di', 'N'), + RangeKeyColumn('photo_id', 'pi', 'S'), + SetColumn('asset_keys', 'ak', 'SS')]), + + # The USER_POST is associated with a POST object, and represents + # user-specific override of information in the POST. 'timestamp' + # records the creation time of the record, and 'labels' contains + # a set of values which describes the customizations. For example, + # the 'removed' label indicates that the post should not be shown + # in the user's personal collection. + # + # Rows in the USER_POST table are only created if the user wants + # to customize the viewpoint in some way. In the absence of a + # row, default values are assumed. + IndexedTable(USER_POST, 'uo', read_units=400, write_units=10, + columns=[HashKeyColumn('user_id', 'ui', 'N'), + RangeKeyColumn('post_id', 'pi', 'S'), + Column('timestamp', 'ti', 'N'), + SetColumn('labels', 'la', 'SS')]), + + # Key is viewpoint-id. Viewpoints are a collection of episodes. + # Viewpoint title and description are indexed for full-text + # search. The viewpoint name, sort of like a twitter + # handler, is also indexed. 'type' is one of: + # ('default', 'event', 'thematic') + # + # The 'update_seq' is incremented each time a viewpoint asset + # is added, removed, or updated. Using this with the 'viewed_seq' + # attribute on Follower, clients can easily determine if there + # is any "unread" content in the viewpoint. Note that updates to + # user-specific content on Follower does not trigger the increment + # of this value. 'last_updated' is set to the creation timestamp of + # the latest activity that was added to this viewpoint. + # + # The 'cover_photo' column is a JSON-encoded dict of photo_id and + # episode_id which indicates which photo should be used as the cover + # photo for the viewpoint. An absent column or None value for this indicates + # that it's explicitly not available (no visible photos in the viewpoint). + # Default viewpoints will not have this column set. + IndexedTable(VIEWPOINT, 'vp', read_units=400, write_units=10, + columns=[HashKeyColumn('viewpoint_id', 'vi', 'S'), + Column('user_id', 'ui', 'N', SecondaryIndexer(), read_only=True), + Column('timestamp', 'ts', 'N', read_only=True), + Column('title', 'ti', 'S', + FullTextIndexer(metaphone=Indexer.Option.YES)), + Column('description', 'de', 'S', + FullTextIndexer(metaphone=Indexer.Option.YES)), + Column('last_updated', 'lu', 'N'), + Column('name', 'na', 'S', SecondaryIndexer()), + Column('type', 'ty', 'S', read_only=True), + Column('update_seq', 'us', 'N'), + JSONColumn('cover_photo', 'cp')]), + + # The index table for all indexed terms. Maps from a string to a + # string for greatest flexibility. This requires that the various + # database objects convert from a string value if the doc-id + # actually does represent a number, such as the user-id in some of + # the indexed tables in this schema. + IndexTable(INDEX, 'S', 'S', read_units=200, write_units=50), + + # For the dynamodb_client_test. + Table(TEST_RENAME, 'test', read_units=10, write_units=5, name_in_db="Test", + columns=[HashKeyColumn('test_hk', 'thk', 'S'), + RangeKeyColumn('test_rk', 'trk', 'N'), + Column('attr0', 'a0', 'N'), + Column('attr1', 'a1', 'N'), + Column('attr2', 'a2', 'S'), + Column('attr3', 'a3', 'NS'), + Column('attr4', 'a4', 'SS')]), + ]) diff --git a/backend/db/viewpoint.py b/backend/db/viewpoint.py new file mode 100644 index 0000000..e424828 --- /dev/null +++ b/backend/db/viewpoint.py @@ -0,0 +1,531 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Viewfinder viewpoint. + +Viewpoints are collections of episodes. Every user has a 'default' +viewpoint which contains all uploaded episodes. Viewpoints of type +'event' are created when a user shares episodes. Additional viewpoints +of type 'thematic' may be created to encompass arbitrary content. For +example, a shared set of family events, funny things you've seen in +NYC, all concerts at an event space, or a photographer's fashion +photos. + +Viewpoint ids are constructed from a variable-length-encoded integer +device id and a variable-length-encoded unique id from the device. The +final value is base64-hex encoded. + +Viewpoints can have followers, which are users who have permission to +view and possibly modify the viewpoint's content. Episodes are added +to a viewpoint via the 'Episode' relation. + +Viewpoint types include: + + 'default': every user has a default viewpoint to which all uploaded + episodes are published. + + 'event': event viewpoints are created every time an episode is shared. + The sharees are added to the viewpoint as followers. + + 'system': system-generated viewpoints used to welcome new users. + + Viewpoint: aggregation of episodes. +""" + +__authors__ = ['andy@emailscrubbed.com (Andy Kimball)', + 'spencer@emailscrubbed.com (Spencer Kimball)'] + +import json + +from tornado import gen +from viewfinder.backend.db import db_client, vf_schema +from viewfinder.backend.db.activity import Activity +from viewfinder.backend.db.asset_id import IdPrefix, ConstructAssetId, DeconstructAssetId, VerifyAssetId +from viewfinder.backend.db.base import DBObject +from viewfinder.backend.db.comment import Comment +from viewfinder.backend.db.hash_base import DBHashObject +from viewfinder.backend.db.friend import Friend +from viewfinder.backend.db.followed import Followed +from viewfinder.backend.db.follower import Follower +from viewfinder.backend.db.lock import Lock +from viewfinder.backend.db.lock_resource_type import LockResourceType +from viewfinder.backend.db.operation import Operation +from viewfinder.backend.db.viewpoint_lock_tracker import ViewpointLockTracker + + +@DBObject.map_table_attributes +class Viewpoint(DBHashObject): + """Viewfinder viewpoint data object.""" + __slots__ = [] + + _table = DBObject._schema.GetTable(vf_schema.VIEWPOINT) + + DEFAULT = 'default' + EVENT = 'event' + SYSTEM = 'system' + + TYPES = [DEFAULT, EVENT, SYSTEM] + """Kinds of viewpoints.""" + + # Limit how many followers may be part of a viewpoint. + # Any change to this value should be coordinated with viewfinder client code + # to ensure that our clients catch this condition before sending to the server. + MAX_FOLLOWERS = 150 + + # Attributes that are projected for removed viewpoints. + _IF_REMOVED_ATTRIBUTES = set(['viewpoint_id', + 'type', + 'follower_id', + 'user_id', + 'timestamp', + 'labels', + 'adding_user_id']) + + def __init__(self, viewpoint_id=None): + super(Viewpoint, self).__init__() + self.viewpoint_id = viewpoint_id + + @classmethod + def ShouldScrubColumn(cls, name): + return name == 'title' + + def IsDefault(self): + """Returns true if the viewpoint is a default viewpoint.""" + return self.type == Viewpoint.DEFAULT + + def IsSystem(self): + """Returns true if the viewpoint is a system viewpoint (ex. welcome conversation).""" + return self.type == Viewpoint.SYSTEM + + @classmethod + def ConstructViewpointId(cls, device_id, uniquifier): + """Returns a viewpoint id constructed from component parts. See + "ConstructAssetId" for details of the encoding. + """ + return ConstructAssetId(IdPrefix.Viewpoint, device_id, uniquifier) + + @classmethod + def DeconstructViewpointId(cls, viewpoint_id): + """Returns the components of a viewpoint id: device_id and + uniquifier. + """ + return DeconstructAssetId(IdPrefix.Viewpoint, viewpoint_id) + + @classmethod + def ConstructCoverPhoto(cls, episode_id, photo_id): + """Construct a cover_photo dict.""" + assert episode_id is not None, episode_id + assert photo_id is not None, photo_id + return {'episode_id': episode_id, 'photo_id': photo_id} + + @classmethod + @gen.coroutine + def VerifyViewpointId(cls, client, user_id, device_id, viewpoint_id): + """Ensures that a client-provided viewpoint id is valid according + to the rules specified in VerifyAssetId. + """ + yield VerifyAssetId(client, user_id, device_id, IdPrefix.Viewpoint, viewpoint_id, has_timestamp=False) + + @classmethod + @gen.engine + def AcquireLock(cls, client, viewpoint_id, callback): + """Acquires a persistent global lock on the specified viewpoint.""" + op = Operation.GetCurrent() + lock = yield gen.Task(Lock.Acquire, client, LockResourceType.Viewpoint, viewpoint_id, + op.operation_id) + ViewpointLockTracker.AddViewpointId(viewpoint_id) + callback(lock) + + @classmethod + @gen.engine + def ReleaseLock(cls, client, viewpoint_id, lock, callback): + """Releases a previously acquired lock on the specified viewpoint.""" + yield gen.Task(lock.Release, client) + ViewpointLockTracker.RemoveViewpointId(viewpoint_id) + callback() + + @classmethod + def AssertViewpointLockAcquired(cls, viewpoint_id): + """Asserts that a lock has been acquired on the specified viewpoint.""" + assert ViewpointLockTracker.HasViewpointId(viewpoint_id), \ + 'Lock for viewpoint, %s, should be acquired at this point but isn\'t.' % viewpoint_id + + def IsCoverPhotoSet(self): + """The cover photo is consider set if it is a non empty dict.""" + if self.cover_photo is not None: + assert len(self.cover_photo) > 0, self + return True + return False + + def MakeMetadataDict(self, follower): + """Constructs a dictionary containing viewpoint metadata attributes, overridden by follower + attributes where required (as viewed by the follower himself). The format conforms to + VIEWPOINT_METADATA in json_schema.py. + """ + # Combine all attributes from the viewpoint and follower records. + vp_dict = self._asdict() + foll_dict = follower.MakeMetadataDict() + vp_dict.update(foll_dict) + + # If the follower is removed from the viewpoint, then only project certain attributes. + if follower.IsRemoved(): + for attr_name in vp_dict.keys(): + if attr_name not in Viewpoint._IF_REMOVED_ATTRIBUTES: + del vp_dict[attr_name] + + return vp_dict + + @gen.coroutine + def AddFollowers(self, client, adding_user_id, existing_follower_ids, add_follower_ids, timestamp): + """Adds the specified followers to this viewpoint, giving each follower CONTRIBUTE + permission on the viewpoint. The caller is responsible for ensuring that the user adding + the followers has permission to do so, and that the users to add are not yet followers. + Returns the newly added followers. + """ + @gen.coroutine + def _UpdateFollower(follower_id): + """Create a new follower of this viewpoint in the database.""" + follower = Follower(user_id=follower_id, viewpoint_id=self.viewpoint_id) + follower.timestamp = timestamp + follower.adding_user_id = adding_user_id + follower.viewed_seq = 0 + follower.labels = [Follower.CONTRIBUTE] + + # Create the follower and corresponding Followed record. + yield [gen.Task(follower.Update, client), + gen.Task(Followed.UpdateDateUpdated, client, follower_id, self.viewpoint_id, + old_timestamp=None, new_timestamp=timestamp)] + + raise gen.Return(follower) + + # Adding user should be an existing user. + assert adding_user_id is None or adding_user_id in existing_follower_ids, \ + (adding_user_id, existing_follower_ids) + + # Caller should never pass overlapping existing/add user id sets. + assert not any(follower_id in existing_follower_ids for follower_id in add_follower_ids), \ + (existing_follower_ids, add_follower_ids) + + # Ensure that friendships are created between the followers to add. + yield gen.Task(Friend.MakeFriendsWithGroup, client, add_follower_ids) + + # Ensure that friendships are created with existing followers. + yield [gen.Task(Friend.MakeFriends, client, existing_id, add_id) + for existing_id in existing_follower_ids + for add_id in add_follower_ids] + + # Add new followers to viewpoint with CONTRIBUTE permission. + add_followers = yield [_UpdateFollower(follower_id) for follower_id in add_follower_ids] + + raise gen.Return(add_followers) + + @gen.engine + def SelectCoverPhoto(self, client, exclude_posts_set, callback, activities_list=None, available_posts_dict=None): + """Select a cover photo for this viewpoint. + This is used to select a cover photo if the current cover photo gets unshared. + The selection order here assumes that the order of episodes and photos in the + activities reflects the intended order of selection. This won't be true + of activities created before this change goes into production, but we've + decided to accept this small variation in cover photo selection for these + older activities because we don't think it's worth the extra complexity that it + would take to make selection of those more 'correct'. + Newer clients should order episodes and photos within share requests according to + cover photo selection priority. + Although older clients provide un-ordered lists of episodes/photos, a request transform + will order episodes/photos from those clients using the original mobile client + algorithm for cover photo selection. So we will select a new cover photo + assuming these activities are already ordered. + Caller may supply list of activities to use so that querying them for the viewpoint isn't needed. + Caller may supply dict of available (not Removed/Unshared) posts so querying for posts isn't needed. + Search process: + 1) oldest to newest activity (share_new or share_existing activities). + 2) within activity, first to last episode. + 3) within episode, first to last photo. + Only photos which aren't unshared qualify. + Returns: cover_photo dict of selected photo or None if one if no selection is found. + """ + from viewfinder.backend.db.post import Post + batch_limit = 50 + + # cover_photo is not supported on default viewpoints. + assert not self.IsDefault(), self + + @gen.coroutine + def _QueryAvailablePost(episode_id, photo_id): + if available_posts_dict is not None: + post = available_posts_dict.get(Post.ConstructPostId(episode_id, photo_id), None) + else: + post = yield gen.Task(Post.Query, client, episode_id, photo_id, col_names=None) + if post.IsRemoved(): + post = None + raise gen.Return(post) + + # Loop over activities starting from the oldest. + excl_start_key = None + while True: + if activities_list is None: + activities = yield gen.Task(Activity.RangeQuery, + client, + self.viewpoint_id, + range_desc=None, + limit=batch_limit, + col_names=None, + excl_start_key=excl_start_key, + scan_forward=False) + else: + activities = activities_list + + for activity in activities: + if activity.name == 'share_new' or activity.name == 'share_existing': + args_dict = json.loads(activity.json) + # Now, loop through the episodes in the activity. + for ep_dict in args_dict['episodes']: + episode_id = ep_dict['episode_id'] + # And loop through the photos in each episode. + for photo_id in ep_dict['photo_ids']: + # Save cost of query on a post that we know is UNSHARED. + if (episode_id, photo_id) not in exclude_posts_set: + post = yield _QueryAvailablePost(episode_id, photo_id) + if post is not None: + # If it hasn't been unshared, we're good to go. + callback(Viewpoint.ConstructCoverPhoto(episode_id, photo_id)) + return + + if activities_list is not None or len(activities) < batch_limit: + break + else: + excl_start_key = activities[-1].GetKey() + + # No unshared photos found in this viewpoint. + callback(None) + + @classmethod + def SelectCoverPhotoFromEpDicts(cls, ep_dicts): + """Select a cover photo from the ep_dicts argument. + Selection assumes episodes and photos are ordered according to selection preference. + Returns: Either None if no photos found, or a cover_photo dict with selected photo. + """ + cover_photo = None + for ep_dict in ep_dicts: + if len(ep_dict['photo_ids']) > 0: + cover_photo = Viewpoint.ConstructCoverPhoto(ep_dict['episode_id'], ep_dict['photo_ids'][0]) + break + return cover_photo + + @classmethod + def IsCoverPhotoContainedInEpDicts(cls, cover_episode_id, cover_photo_id, ep_dicts): + """Confirm existence of specified cover_photo in ep_dicts. + Return: True if specified cover_photo matches photo in ep_dicts. Otherwise, False.""" + for ep_dict in ep_dicts: + if cover_episode_id == ep_dict['episode_id']: + for photo_id in ep_dict['photo_ids']: + if cover_photo_id == photo_id: + return True + # Not found. + return False + + @classmethod + @gen.coroutine + def CreateDefault(cls, client, user_id, device_id, timestamp): + """Creates and returns a new user's default viewpoint.""" + from viewfinder.backend.db.user import User + vp_dict = {'viewpoint_id': Viewpoint.ConstructViewpointId(device_id, User.DEFAULT_VP_ASSET_ID), + 'user_id': user_id, + 'timestamp': timestamp, + 'type': Viewpoint.DEFAULT} + viewpoint, _ = yield gen.Task(Viewpoint.CreateNew, client, **vp_dict) + raise gen.Return(viewpoint) + + @classmethod + @gen.coroutine + def CreateNew(cls, client, **vp_dict): + """Creates the viewpoint specified by 'vp_dict' and creates a follower relation between + the requesting user and the viewpoint with the ADMIN label. The caller is responsible for + checking permission to do this, as well as ensuring that the viewpoint does not yet exist + (or is just being identically rewritten). + + Returns a tuple containing the newly created objects: (viewpoint, follower). + """ + tasks = [] + + # Create the viewpoint. + assert 'viewpoint_id' in vp_dict and 'user_id' in vp_dict and 'timestamp' in vp_dict, vp_dict + viewpoint = Viewpoint.CreateFromKeywords(**vp_dict) + viewpoint.last_updated = viewpoint.timestamp + viewpoint.update_seq = 0 + tasks.append(gen.Task(viewpoint.Update, client)) + + # Create the follower and give all permissions, since it's the owner. + foll_dict = {'user_id': viewpoint.user_id, + 'viewpoint_id': viewpoint.viewpoint_id, + 'timestamp': viewpoint.timestamp, + 'labels': list(Follower.PERMISSION_LABELS), + 'viewed_seq': 0} + if viewpoint.IsDefault(): + foll_dict['labels'].append(Follower.PERSONAL) + + follower = Follower.CreateFromKeywords(**foll_dict) + tasks.append(gen.Task(follower.Update, client)) + + # Create the corresponding Followed record. + tasks.append(gen.Task(Followed.UpdateDateUpdated, + client, + viewpoint.user_id, + viewpoint.viewpoint_id, + old_timestamp=None, + new_timestamp=viewpoint.last_updated)) + yield tasks + + raise gen.Return((viewpoint, follower)) + + @classmethod + @gen.coroutine + def CreateNewWithFollowers(cls, client, follower_ids, **vp_dict): + """Calls the "CreateWithFollower" method to create a viewpoint with a single follower + (the current user). Then, all users identified by "follower_ids" are added to that + viewpoint as followers. Ensure that every pair of followers is friends with each other. + The caller is responsible for checking permission to do this, as well as ensuring that + the viewpoint and followers do not yet exist (or are just being identically rewritten). + + Returns a tuple containing the newly created objects: (viewpoint, followers). The + followers list includes the owner. + """ + # Create the viewpoint with the current user as its only follower. + viewpoint, owner_follower = yield Viewpoint.CreateNew(client, **vp_dict) + + # Now add the additional followers. + followers = yield viewpoint.AddFollowers(client, + vp_dict['user_id'], + [vp_dict['user_id']], + follower_ids, + viewpoint.timestamp) + followers.append(owner_follower) + + raise gen.Return((viewpoint, followers)) + + @classmethod + @gen.coroutine + def QueryWithFollower(cls, client, user_id, viewpoint_id): + """Queries the specified viewpoint and follower and returns them as a (viewpoint, follower) + tuple. + """ + viewpoint, follower = yield [gen.Task(Viewpoint.Query, client, viewpoint_id, None, must_exist=False), + gen.Task(Follower.Query, client, user_id, viewpoint_id, None, must_exist=False)] + assert viewpoint is not None or follower is None, (viewpoint, follower) + raise gen.Return((viewpoint, follower)) + + @classmethod + @gen.engine + def QueryEpisodes(cls, client, viewpoint_id, callback, excl_start_key=None, limit=None): + """Queries episodes belonging to the viewpoint (up to 'limit' total) for + the specified 'viewpoint_id'. Starts with episodes having a key greater + than 'excl_start_key'. Returns a tuple with the array of episodes and + the last queried key. + """ + from viewfinder.backend.db.episode import Episode + + # Query the viewpoint_id secondary index with excl_start_key & limit. + query_expr = ('episode.viewpoint_id={id}', {'id': viewpoint_id}) + start_index_key = db_client.DBKey(excl_start_key, None) if excl_start_key is not None else None + episode_keys = yield gen.Task(Episode.IndexQueryKeys, client, query_expr, + start_index_key=start_index_key, limit=limit) + episodes = yield gen.Task(Episode.BatchQuery, client, episode_keys, None) + callback((episodes, episode_keys[-1].hash_key if len(episode_keys) > 0 else None)) + + @classmethod + @gen.coroutine + def QueryFollowers(cls, client, viewpoint_id, excl_start_key=None, limit=None): + """Query followers belonging to the viewpoint (up to 'limit' total) for + the specified 'viewpoint_id'. The query is for followers starting with + (but excluding) 'excl_start_key'. The callback is invoked with an array + of follower objects and the last queried key. + """ + # Query the viewpoint_id secondary index with excl_start_key & limit. + query_expr = ('follower.viewpoint_id={id}', {'id': viewpoint_id}) + start_index_key = db_client.DBKey(excl_start_key, viewpoint_id) if excl_start_key is not None else None + follower_keys = yield gen.Task(Follower.IndexQueryKeys, + client, + query_expr, + start_index_key=start_index_key, + limit=limit) + + last_key = follower_keys[-1].hash_key if len(follower_keys) > 0 else None + + followers = yield gen.Task(Follower.BatchQuery, client, follower_keys, None) + + raise gen.Return((followers, last_key)) + + @classmethod + def QueryFollowerIds(cls, client, viewpoint_id, callback, excl_start_key=None, limit=None): + """Query followers belonging to the viewpoint (up to 'limit' total) for + the specified 'viewpoint_id'. The query is for followers starting with + (but excluding) 'excl_start_key'. The callback is invoked with an array + of follower user ids and the last queried key. + """ + def _OnQueryFollowerKeys(follower_keys): + follower_ids = [key.hash_key for key in follower_keys] + last_key = follower_ids[-1] if len(follower_ids) > 0 else None + + callback((follower_ids, last_key)) + + # Query the viewpoint_id secondary index with excl_start_key & limit. + query_expr = ('follower.viewpoint_id={id}', {'id': viewpoint_id}) + start_index_key = db_client.DBKey(excl_start_key, viewpoint_id) if excl_start_key is not None else None + Follower.IndexQueryKeys(client, query_expr, callback=_OnQueryFollowerKeys, + start_index_key=start_index_key, limit=limit) + + @classmethod + def VisitFollowerIds(cls, client, viewpoint_id, visitor, callback, consistent_read=False): + """Visit all followers of the specified viewpoint and invoke the + "visitor" function with each follower id. See VisitIndexKeys for + additional detail. + """ + def _OnVisit(follower_key, visit_callback): + visitor(follower_key.hash_key, visit_callback) + + query_expr = ('follower.viewpoint_id={id}', {'id': viewpoint_id}) + Follower.VisitIndexKeys(client, query_expr, _OnVisit, callback, consistent_read=consistent_read) + + @classmethod + def QueryActivities(cls, client, viewpoint_id, callback, excl_start_key=None, limit=None): + """Queries activities belonging to the viewpoint (up to 'limit' total) for + the specified 'viewpoint_id'. Starts with activities having a key greater + than 'excl_start_key'. Returns a tuple with the array of activities and + the last queried key. + """ + def _OnQueryActivities(activities): + callback((activities, activities[-1].activity_id if len(activities) > 0 else None)) + + Activity.RangeQuery(client, viewpoint_id, range_desc=None, limit=limit, col_names=None, + callback=_OnQueryActivities, excl_start_key=excl_start_key) + + @classmethod + def QueryComments(cls, client, viewpoint_id, callback, excl_start_key=None, limit=None): + """Queries comments belonging to the viewpoint (up to 'limit' total) for + the specified 'viewpoint_id'. Starts with comments having a key greater + than 'excl_start_key'. Returns a tuple with the array of comments and + the last queried key. + """ + def _OnQueryComments(comments): + callback((comments, comments[-1].comment_id if len(comments) > 0 else None)) + + Comment.RangeQuery(client, viewpoint_id, range_desc=None, limit=limit, col_names=None, + callback=_OnQueryComments, excl_start_key=excl_start_key) + + @classmethod + @gen.engine + def AddFollowersOperation(cls, client, callback, activity, user_id, viewpoint_id, contacts): + """Adds contacts as followers to the specified viewpoint. Notifies all viewpoint + followers about the new followers. + """ + # TODO(Andy): Remove this once the AddFollowersOperation is in production. + from viewfinder.backend.op.add_followers_op import AddFollowersOperation + AddFollowersOperation.Execute(client, activity, user_id, viewpoint_id, contacts, callback=callback) + + @classmethod + @gen.engine + def UpdateOperation(cls, client, callback, act_dict, vp_dict): + """Updates viewpoint metadata.""" + # TODO(Andy): Remove this once the UpdateViewpointOperation is in production. + from viewfinder.backend.op.update_viewpoint_op import UpdateViewpointOperation + user_id = vp_dict.pop('user_id') + UpdateViewpointOperation.Execute(client, act_dict, user_id, vp_dict, callback=callback) diff --git a/backend/db/viewpoint_lock_tracker.py b/backend/db/viewpoint_lock_tracker.py new file mode 100644 index 0000000..990e21c --- /dev/null +++ b/backend/db/viewpoint_lock_tracker.py @@ -0,0 +1,54 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Tracks set of viewpoints locked by an operation. + +Certain actions require viewpoint lock(s) to be acquired before it's safe to perform them. +For example, if a user adds a follower to a viewpoint, that viewpoint should be locked in +order to avoid conflicts with other users. The ViewpointLockTracker keeps a list of +viewpoints that have been locked by the current Operation. This allows us to ensure that +viewpoints are not locked multiple times, and also to assert that certain viewpoint(s) +have been locked before running code that requires those locks. +""" + +from viewfinder.backend.db.operation import Operation + + +class ViewpointLockTracker(object): + """Container to help track which viewpoints have locks acquired for them. This will help + assert that we have the proper viewpoint locks before modifying them. + """ + def __init__(self): + self.viewpoint_lock_ids = set() + + @classmethod + def AddViewpointId(cls, viewpoint_lock_id): + """Adds a viewpoint id to the set on the current operation.""" + lock_tracker = ViewpointLockTracker._GetInstance() + assert viewpoint_lock_id not in lock_tracker.viewpoint_lock_ids + lock_tracker.viewpoint_lock_ids.add(viewpoint_lock_id) + + @classmethod + def RemoveViewpointId(cls, viewpoint_lock_id): + """Removes a viewpoint id from the set on the current operation.""" + lock_tracker = ViewpointLockTracker._GetInstance() + assert viewpoint_lock_id in lock_tracker.viewpoint_lock_ids + lock_tracker.viewpoint_lock_ids.remove(viewpoint_lock_id) + + @classmethod + def HasViewpointId(cls, viewpoint_lock_id): + """Returns true if the the viewpoint id is in the set on the current operation.""" + lock_tracker = ViewpointLockTracker._GetInstance() + return viewpoint_lock_id in lock_tracker.viewpoint_lock_ids + + @classmethod + def _GetInstance(cls): + """Ensures that a viewpoint lock tracker has been created and attached to the current + operation. + """ + op = Operation.GetCurrent() + lock_tracker = op.context.get('viewpoint_lock_tracker') + if lock_tracker is None: + lock_tracker = ViewpointLockTracker() + op.context['viewpoint_lock_tracker'] = lock_tracker + + return lock_tracker diff --git a/backend/logs/README b/backend/logs/README new file mode 100644 index 0000000..3c0a4db --- /dev/null +++ b/backend/logs/README @@ -0,0 +1,10 @@ +This directory contains all modules falling in the broad category of "logs analysis". +This includes: +- get_server_logs: iterates over 'full' backend logs and merges them per day and instance +- analyze_merged_logs: iterates over merged 'full' backend logs and computes various statistics +- itunes_trends: fetches daily stats from iTunesConnect +- get_table_sizes: looks up dynamodb table sizes + +These jobs all touch common stats: S3 logs or dynamodb metrics table. +They all require individual locks to prevent concurrent instances from running. +They are run serially by 'run_logs_analysis.py' to guarantee order and prevent concurrent runs. diff --git a/backend/logs/__init__.py b/backend/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/logs/analyze_analytics_logs.py b/backend/logs/analyze_analytics_logs.py new file mode 100644 index 0000000..8c85f07 --- /dev/null +++ b/backend/logs/analyze_analytics_logs.py @@ -0,0 +1,255 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Run analysis over all merged user analytics logs. + +Computes speed percentiles for full asset scans (only those lasting more than 1s for more accurate numbers). + +Automatically finds the list of merged logs in S3. If --start_date=YYYY-MM-DD is specified, only analyze logs +starting from a week before that date (we give user logs that much time to get uploaded). + + +Usage: +# Analyze all logs. +python -m viewfinder.backend.logs.analyze_analytics_logs + +# Analyze logs from a specific start date. +python -m viewfinder.backend.logs.analyze_analytics_logs --start_date=2012-12-15 + +Other options: +-require_lock: default=True: hold the job:analyze_analytics lock during processing. +-smart_scan: default=False: determine the start date from previous run summaries. +-hours_between_runs: default=0: don't run if last successful run started less than this many hours ago. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import cStringIO +import json +import logging +import numpy +import os +import sys +import time +import traceback + +from collections import defaultdict, Counter +from tornado import gen, options +from viewfinder.backend.base import constants, main, statistics, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util +from viewfinder.backend.storage.object_store import ObjectStore +from viewfinder.backend.storage import store_utils + +# TODO(marc): automatic date detection (eg: find latest metric entry and process from 30 days before). +options.define('start_date', default=None, help='Start date (filename start key). May be overridden by smart_scan.') +options.define('dry_run', default=True, help='Do not update dynamodb metrics table') +options.define('compute_today', default=False, help='Do not compute statistics for today, logs will be partial') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:analyze_analytics lock before running. Exit if acquire fails.') +options.define('smart_scan', type=bool, default=False, + help='determine start_date from previous successful runs.') +options.define('hours_between_runs', type=int, default=0, + help='minimum time since start of last successful run (with dry_run=False)') + +class DayStats(object): + def __init__(self, day): + self.day = day + self._scan_durations = [] + self._long_scan_speeds = [] + self._photos_scanned = [] + # Number of unique users recording an event on this day. + self.event_users = Counter() + # Number of occurrences of an event aggregated across all users. + self.total_events = Counter() + + def AddScan(self, version, photos, duration): + self._scan_durations.append(duration) + self._photos_scanned.append(photos) + if duration > 1.0: + self._long_scan_speeds.append(photos / duration) + + def AddEvents(self, counters): + for name, count in counters.iteritems(): + self.total_events[name] += count + self.event_users[name] += 1 + + def PrintSummary(self): + logging.info('Day: %s\n %s' % (self.day, statistics.FormatStats(self._long_scan_speeds, percentiles=[90,95,99]))) + + def ScanDurationPercentile(self, percentile): + return numpy.percentile(self._scan_durations, percentile) + + def LongScanSpeedPercentile(self, percentile): + return numpy.percentile(self._long_scan_speeds, percentile) + + def PhotosScannedPercentile(self, percentile): + return numpy.percentile(self._photos_scanned, percentile) + + +@gen.engine +def ProcessFiles(merged_store, filenames, callback): + """Fetch and process each file contained in 'filenames'.""" + + @gen.engine + def _ProcessOneFile(contents, day_stats): + """Iterate over the contents of a processed file: one entry per line. Increment stats for specific entries.""" + buf = cStringIO.StringIO(contents) + buf.seek(0) + ui_events = Counter() + while True: + line = buf.readline() + if not line: + break + parsed = json.loads(line) + if not parsed: + continue + if 'version' not in parsed: + continue + # TODO(marc): lookup the user's device ID in dynamodb and get device model. + payload = parsed['payload'] + if 'name' in payload: + if payload['name'] == '/assets/scan' and payload['type'] == 'full': + day_stats.AddScan(parsed['version'], payload['num_scanned'], payload['elapsed']) + elif payload['name'].startswith('/ui/'): + ui_events[payload['name']] += 1 + if ui_events: + ui_events['/ui/anything'] += 1 + day_stats.AddEvents(ui_events) + buf.close() + + today = util.NowUTCToISO8601() + # Group filenames by day. + files_by_day = defaultdict(list) + for filename in filenames: + _, day, user = filename.split('/') + if options.options.compute_today or today != day: + files_by_day[day].append(filename) + + # Compute per-day totals. Toss them into a list, we'll want it sorted. + stats_by_day = {} + for day in sorted(files_by_day.keys()): + # We don't really need to process days in-order, but it's nicer. + files = files_by_day[day] + day_stats = DayStats(day) + for f in files: + contents = '' + try: + contents = yield gen.Task(merged_store.Get, f) + except Exception as e: + logging.error('Error fetching file %s: %r' % (f, e)) + continue + _ProcessOneFile(contents, day_stats) + if len(day_stats._long_scan_speeds) == 0: + continue + dd = DotDict() + for p in [1, 5, 10, 25, 50, 75, 90, 95, 99]: + dd['user_analytics.scans_gt1s_speed_percentile.%.2d' % p] = day_stats.LongScanSpeedPercentile(p) + dd['user_analytics.scans_duration_percentile.%.2d' % p] = day_stats.ScanDurationPercentile(p) + dd['user_analytics.scans_num_photos_percentile.%.2d' % p] = day_stats.PhotosScannedPercentile(p) + dd['user_analytics.ui.event_users'] = day_stats.event_users + dd['user_analytics.ui.total_events'] = day_stats.total_events + stats_by_day[day] = dd + + callback(stats_by_day) + +@gen.engine +def GetMergedLogsFileList(merged_store, marker, callback): + """Fetch the list of file names from S3.""" + registry_dir = os.path.join(logs_util.UserAnalyticsLogsPaths.kMergedLogsPrefix, + logs_util.UserAnalyticsLogsPaths.kRegistryDir) + def _WantFile(filename): + return not filename.startswith(registry_dir) + + base_path = logs_util.UserAnalyticsLogsPaths.kMergedLogsPrefix + '/' + marker = os.path.join(base_path, marker) if marker is not None else None + file_list = yield gen.Task(store_utils.ListAllKeys, merged_store, prefix=base_path, marker=marker) + files = [f for f in file_list if _WantFile(f)] + files.sort() + + logging.info('found %d merged log files, analyzing %d' % (len(file_list), len(files))) + callback(files) + + +@gen.engine +def RunOnce(client, job, callback): + """Get list of files and call processing function.""" + merged_store = ObjectStore.GetInstance(logs_util.UserAnalyticsLogsPaths.MERGED_LOGS_BUCKET) + + start_date = options.options.start_date + if options.options.smart_scan: + # Search for successful full-scan run in the last week. + last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day') + if last_run is None: + logging.info('No previous successful scan found, rerun with --start_date') + callback(None) + return + + last_run_start = last_run['start_time'] + if util.HoursSince(last_run_start) < options.options.hours_between_runs: + logging.info('Last successful run started at %s, less than %d hours ago; skipping.' % + (time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs)) + callback(None) + return + + last_day = last_run['stats.last_day'] + # Set scan_start to start of previous run - 30d (we need 30 days' worth of data to properly compute + # 30-day active users. Add an extra 3 days just in case we had some missing logs during the last run. + start_time = util.ISO8601ToUTCTimestamp(last_day, hour=12) - constants.SECONDS_PER_WEEK + start_date = util.TimestampUTCToISO8601(start_time) + logging.info('Last successful analyze_analytics run (%s) scanned up to %s, setting analysis start date to %s' % + (time.asctime(time.localtime(last_run_start)), last_day, start_date)) + + # Fetch list of merged logs. + files = yield gen.Task(GetMergedLogsFileList, merged_store, start_date) + day_stats = yield gen.Task(ProcessFiles, merged_store, files) + + # Write per-day stats to dynamodb. + if len(day_stats) > 0: + hms = logs_util.kDailyMetricsTimeByLogType['analytics_logs'] + yield gen.Task(logs_util.UpdateMetrics, client, day_stats, dry_run=options.options.dry_run, hms_tuple=hms) + last_day = sorted(day_stats.keys())[-1] + callback(last_day) + else: + callback(None) + +@gen.engine +def _Start(callback): + """Grab a lock on job:analyze_analytics and call RunOnce. If we get a return value, write it to the job summary.""" + client = db_client.DBClient.Instance() + job = Job(client, 'analyze_analytics') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + result = None + job.Start() + try: + result = yield gen.Task(RunOnce, client, job) + except: + # Failure: log run summary with trace. + typ, val, tb = sys.exc_info() + msg = ''.join(traceback.format_exception(typ, val, tb)) + logging.info('Registering failed run with message: %s' % msg) + yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) + else: + if result is not None and not options.options.dry_run: + # Successful run with data processed and not in dry-run mode: write run summary. + stats = DotDict() + stats['last_day'] = result + logging.info('Registering successful run with stats: %r' % stats) + yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/analyze_dynamodb.py b/backend/logs/analyze_dynamodb.py new file mode 100644 index 0000000..f598f4d --- /dev/null +++ b/backend/logs/analyze_dynamodb.py @@ -0,0 +1,238 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved +"""Crawl dynamodb tables and compute various metrics. + +Run with: +$ python -m viewfinder.backend.logs.analyze_dynamodb --dry_run=False + +Options: +- dry_run: default=True: run in dry-run mode (don't write to dynamodb) +- require_lock: default=True: grab the job:analyze_dynamodb lock for the duration of the job. +- throttling_factor: default=4: set allowed dynamodb capacity to "total / factor" +- force_recompute: default=False: recompute today's stats even if we already have +""" + +import json +import logging +import sys +import time +import traceback + +from collections import Counter +from tornado import gen, options +from viewfinder.backend.base import main, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, metric, vf_schema +from viewfinder.backend.db.device import Device +from viewfinder.backend.db.job import Job +from viewfinder.backend.db.identity import Identity +from viewfinder.backend.db.settings import AccountSettings +from viewfinder.backend.db.user import User +from viewfinder.backend.logs import logs_util + +options.define('dry_run', default=True, help='Print output only, do not write to metrics table.') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:analyze_dynamodb lock before running. Exit if acquire fails.') +options.define('throttling_factor', default=4, help='Dynamodb throttling factor') +options.define('force_recompute', default=False, help='Recompute even if we have already run today') +options.define('limit_users', default=-1, help='Limit number of users to scan. --dry_run only') + + +@gen.engine +def CountByIdentity(client, user_id, callback): + query_str = 'identity.user_id=%d' % user_id + # We only care about the identity type (the key). + result = yield gen.Task(Identity.IndexQuery, client, query_str, col_names=['key']) + if len(result) == 0: + callback(('NONE', 'NONE')) + return + + type_count = Counter() + for r in result: + identity_type, value = Identity.SplitKey(r.key) + type_count[identity_type[0]] += 1 + count_by_type = '' + types = '' + for k in sorted(type_count.keys()): + count_by_type += '%s%d' % (k, type_count[k]) + types += k + callback((count_by_type, types)) + +@gen.engine +def ProcessUserDevices(client, user_id, callback): + devices = [] + start_key = None + while True: + dev_list = yield gen.Task(Device.RangeQuery, client, user_id, None, None, None, excl_start_key=start_key) + if len(dev_list) == 0: + break + devices.extend(dev_list) + start_key = dev_list[-1].GetKey() + + # The highest app version across all devices for this user. This may not be a device with a push token. + highest_version = None + has_notification = 0 + + for d in devices: + if d.push_token is not None and d.alert_user_id == user_id: + has_notification += 1 + if d.version is not None and (highest_version is None or d.version.split('.') > highest_version.split('.')): + highest_version = d.version + + callback((highest_version, has_notification)) + +@gen.engine +def ProcessTables(client, callback): + user_count = Counter() + locale_count = Counter() + + identity_count = Counter() + identity_types = Counter() + + device_highest_version = Counter() + device_has_notification = Counter() + device_notification_count = Counter() + + settings_email_alerts = Counter() + settings_sms_alerts = Counter() + settings_push_alerts = Counter() + settings_storage = Counter() + settings_marketing = Counter() + + start_key = None + limit = options.options.limit_users if options.options.limit_users > 0 else None + while True: + users, start_key = yield gen.Task(User.Scan, client, None, limit=limit, excl_start_key=start_key) + + for user in users: + if user.IsTerminated(): + # This includes terminated prospective users (pretty rare). + user_count['terminated'] += 1 + continue + elif not user.IsRegistered(): + user_count['prospective'] += 1 + continue + + # From here on out, only registered users are part of the stats. + user_count['registered'] += 1 + + # User locale. + locale_count[user.locale or 'NONE'] += 1 + + # Count of identities by type. + counts, types = yield gen.Task(CountByIdentity, client, user.user_id) + identity_count[counts] += 1 + identity_types[types] += 1 + + # Versions and notification status for user's devices. + highest_version, notification_count = yield gen.Task(ProcessUserDevices, client, user.user_id) + device_highest_version[highest_version.replace('.', '_') if highest_version else 'None'] += 1 + device_notification_count[str(notification_count)] += 1 + if notification_count > 0: + device_has_notification['true'] += 1 + else: + device_has_notification['false'] += 1 + + # Account settings. + settings = yield gen.Task(AccountSettings.QueryByUser, client, user.user_id, None) + settings_email_alerts[settings.email_alerts or 'NA'] += 1 + settings_sms_alerts[settings.sms_alerts or 'NA'] += 1 + settings_push_alerts[settings.push_alerts or 'NA'] += 1 + settings_storage[','.join(settings.storage_options) if settings.storage_options else 'NA'] += 1 + settings_marketing[settings.marketing or 'NA'] += 1 + + if limit is not None: + limit -= len(users) + if limit <= 0: + break + + + if start_key is None: + break + + day_stats = DotDict() + day_stats['dynamodb.user.state'] = user_count + day_stats['dynamodb.user.locale'] = locale_count + day_stats['dynamodb.user.identities'] = identity_count + day_stats['dynamodb.user.identity_types'] = identity_types + day_stats['dynamodb.user.device_highest_version'] = device_highest_version + day_stats['dynamodb.user.device_has_notification'] = device_has_notification + day_stats['dynamodb.user.devices_with_notification'] = device_notification_count + day_stats['dynamodb.user.settings_email_alerts'] = settings_email_alerts + day_stats['dynamodb.user.settings_sms_alerts'] = settings_sms_alerts + day_stats['dynamodb.user.settings_push_alerts'] = settings_push_alerts + day_stats['dynamodb.user.settings_storage'] = settings_storage + day_stats['dynamodb.user.settings_marketing'] = settings_marketing + + callback(day_stats) + + +@gen.engine +def RunOnce(client, job, callback): + """Find last successful run. If there is one from today, abort. Otherwise, run everything.""" + today = util.NowUTCToISO8601() + + last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day') + if last_run is not None and last_run['stats.last_day'] == today and not options.options.force_recompute: + logging.info('Already ran successfully today: skipping. Specify --force_recompute to recompute.') + callback(None) + return + + # Analyze. + day_stats = yield gen.Task(ProcessTables, client) + assert day_stats is not None + + # Write per-day stats to dynamodb. + hms = logs_util.kDailyMetricsTimeByLogType['dynamodb_user'] + yield gen.Task(logs_util.UpdateMetrics, client, {today: day_stats}, dry_run=options.options.dry_run, hms_tuple=hms, + prefix_to_erase='dynamodb.user') + callback(today) + + +@gen.engine +def _Start(callback): + """Grab a lock on job:analyze_dynamodb and call RunOnce. If we get a return value, write it to the job summary.""" + # Setup throttling. + for table in vf_schema.SCHEMA.GetTables(): + table.read_units = max(1, table.read_units // options.options.throttling_factor) + table.write_units = max(1, table.write_units // options.options.throttling_factor) + + client = db_client.DBClient.Instance() + job = Job(client, 'analyze_dynamodb') + + if not options.options.dry_run and options.options.limit_users > 0: + logging.error('--limit_users specified, but not running in dry-run mode. Aborting') + callback() + return + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + result = None + job.Start() + try: + result = yield gen.Task(RunOnce, client, job) + except: + # Failure: log run summary with trace. + typ, val, tb = sys.exc_info() + msg = ''.join(traceback.format_exception(typ, val, tb)) + logging.info('Registering failed run with message: %s' % msg) + yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) + else: + if result is not None and not options.options.dry_run: + # Successful run with data processed and not in dry-run mode: write run summary. + stats = DotDict() + stats['last_day'] = result + logging.info('Registering successful run with stats: %r' % stats) + yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/analyze_merged_logs.py b/backend/logs/analyze_merged_logs.py new file mode 100644 index 0000000..7d1fcf9 --- /dev/null +++ b/backend/logs/analyze_merged_logs.py @@ -0,0 +1,337 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Run analysis over all merged log files stored in S3. + +This processes wanted data and writes output files to S3. + +Data processed: +- per day/per user count of requests by type. stored in S3://serverdata/processed_data/user_requests/YYYY-MM-DD + + +Usage: +# Analyze logs written since the last run (plus an extra two days to catch any stragglers). +python -m viewfinder.backend.logs.analyze_merged_logs --smart_scan=True + +# Analyze logs from a specific start date. +python -m viewfinder.backend.logs.analyze_merged_logs --start_date=2012-12-15 + +Other options: +-ec2_only: default=True: only analyze logs from AWS instances. +-require_lock: default=True: hold the job:analyze_logs lock during processing. +-smart_scan: default=False: determine the start date from previous run summaries. +-hours_between_runs: default=0: don't run if last successful run started less than this many hours ago. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import cStringIO +import json +import logging +import os +import re +import sys +import time +import traceback + +from collections import Counter, defaultdict, deque +from itertools import islice +from tornado import gen, options +from viewfinder.backend.base import constants, main, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util +from viewfinder.backend.storage.object_store import ObjectStore +from viewfinder.backend.storage import store_utils + +options.define('start_date', default=None, help='Start date (filename start key). May be overridden by smart_scan.') +options.define('ec2_only', default=True, help='AWS instances only') +options.define('dry_run', default=True, help='Do not update dynamodb metrics table') +options.define('compute_today', default=False, help='Do not compute statistics for today, logs will be partial') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:analyze_logs lock before running. Exit if acquire fails.') +options.define('smart_scan', type=bool, default=False, + help='determine start_date from previous successful runs.') +options.define('hours_between_runs', type=int, default=0, + help='minimum time since start of last successful run (with dry_run=False)') +options.define('max_days_to_process', default=None, type=int, + help='Process at most this many days (in chronological order)') +options.define('trace_context_num_lines', default=2, type=int, + help='Number of lines of context to save before and after a Trace (eg: 2 before and 2 after)') +options.define('process_op_abort', default=True, type=bool, + help='Process and store traces for OP ABORT lines') +options.define('process_traceback', default=True, type=bool, + help='Process and store Traceback lines') + +kTracebackRE = re.compile(r'Traceback \(most recent call last\)') + +@gen.engine +def ProcessFiles(merged_store, logs_paths, filenames, callback): + """Fetch and process each file contained in 'filenames'.""" + + def _ProcessOneFile(contents, day_stats, device_entries, trace_entries): + """Iterate over the contents of a processed file: one entry per line. Increment stats for specific entries.""" + buf = cStringIO.StringIO(contents) + buf.seek(0) + # Max len is +1 since we include the current line. It allows us to call 'continue' in the middle of the loop. + context_before = deque(maxlen=options.options.trace_context_num_lines + 1) + # Traces that still need "after" context. + pending_traces = [] + def _AddTrace(trace_type, timestamp, module, message): + # context_before also has the current line, so grab only :-1. + trace = {'type': trace_type, + 'timestamp': timestamp, + 'module': module, + 'trace': msg, + 'context_before': list(context_before)[:-1], + 'context_after': []} + if options.options.trace_context_num_lines == 0: + trace_entries.append(trace) + else: + pending_traces.append(trace) + + def _CheckPendingTraces(line): + for t in pending_traces: + t['context_after'].append(line) + while pending_traces and len(pending_traces[0]['context_after']) >= options.options.trace_context_num_lines: + trace_entries.append(pending_traces.pop(0)) + + while True: + line = buf.readline() + if not line: + break + line = line.rstrip('\n') + # The deque automatically pops elements from the front when maxlen is reached. + context_before.append(line) + _CheckPendingTraces(line) + + parsed = logs_util.ParseLogLine(line) + if not parsed: + continue + day, time, module, msg = parsed + timestamp = logs_util.DayTimeStringsToUTCTimestamp(day, time) + + if options.options.process_traceback and re.search(kTracebackRE, line): + _AddTrace('traceback', timestamp, module, msg) + + if module.startswith('user_op_manager:') or module.startswith('operation:'): + # Found op status line. + if msg.startswith('SUCCESS'): + # Success message. eg: SUCCESS: user: xx, device: xx, op: xx, method: xx.yy in xxs + parsed = logs_util.ParseSuccessMsg(msg) + if not parsed: + continue + user, device, op, class_name, method_name = parsed + method = '%s.%s' % (class_name, method_name) + day_stats.ActiveAll(user) + if method in ('Follower.UpdateOperation', 'UpdateFollowerOperation.Execute'): + day_stats.ActiveView(user) + elif method in ('Comment.PostOperation', 'PostCommentOperation.Execute'): + day_stats.ActivePost(user) + elif method in ('Episode.ShareExistingOperation', 'Episode.ShareNewOperation', + 'ShareExistingOperation.Execute', 'ShareNewOperation.Execute'): + day_stats.ActiveShare(user) + elif msg.startswith('EXECUTE'): + # Exec message. eg: EXECUTE: user: xx, device: xx, op: xx, method: xx.yy: + parsed = logs_util.ParseExecuteMsg(msg) + if not parsed: + continue + user, device, op, class_name, method_name, request = parsed + method = '%s.%s' % (class_name, method_name) + if method in ('Device.UpdateOperation', 'User.RegisterOperation', 'RegisterUserOperation.Execute'): + try: + req_dict = eval(request) + device_entries.append({'method': method, 'timestamp': timestamp, 'request': req_dict}) + except Exception as e: + continue + elif msg.startswith('ABORT'): + if options.options.process_op_abort: + # Abort message, save the entire line as well as context. + _AddTrace('abort', timestamp, module, msg) + # FAILURE status is already handled by Traceback processing. + elif module.startswith('base:') and msg.startswith('/ping OK:'): + # Ping message. Extract full request dict. + req_str = logs_util.ParsePingMsg(msg) + if not req_str: + continue + try: + req_dict = json.loads(req_str) + device_entries.append({'method': 'ping', 'timestamp': timestamp, 'request': req_dict}) + except Exception as e: + continue + elif module.startswith('ping:') and msg.startswith('ping OK:'): + # Ping message in new format. Extract full request and response dicts. + (req_str, resp_str) = logs_util.ParseNewPingMsg(msg) + if not req_str or not resp_str: + continue + try: + req_dict = json.loads(req_str) + resp_dict = json.loads(resp_str) + device_entries.append({'method': 'ping', 'timestamp': timestamp, 'request': req_dict, 'response': resp_dict}) + except Exception as e: + continue + + + # No more context. Flush the pending traces into the list. + trace_entries.extend(pending_traces) + buf.close() + + today = util.NowUTCToISO8601() + # Group filenames by day. + files_by_day = defaultdict(list) + for filename in filenames: + day = logs_paths.MergedLogPathToDate(filename) + if not day: + logging.error('filename cannot be parsed as processed log: %s' % filename) + continue + if options.options.compute_today or today != day: + files_by_day[day].append(filename) + + # Sort the list of days. This is important both for --max_days_to_process, and to know the last + # day for which we wrote the file. + day_list = sorted(files_by_day.keys()) + if options.options.max_days_to_process is not None: + day_list = day_list[:options.options.max_days_to_process] + + last_day_written = None + for day in day_list: + files = files_by_day[day] + day_stats = logs_util.DayUserRequestStats(day) + device_entries = [] + trace_entries = [] + for f in files: + # Let exceptions surface. + contents = yield gen.Task(merged_store.Get, f) + logging.info('Processing %d bytes from %s' % (len(contents), f)) + _ProcessOneFile(contents, day_stats, device_entries, trace_entries) + + if not options.options.dry_run: + # Write the json-ified stats. + req_contents = json.dumps(day_stats.ToDotDict()) + req_file_path = 'processed_data/user_requests/%s' % day + dev_contents = json.dumps(device_entries) + dev_file_path = 'processed_data/device_details/%s' % day + try: + trace_contents = json.dumps(trace_entries) + except Exception as e: + trace_contents = None + trace_file_path = 'processed_data/traces/%s' % day + + + @gen.engine + def _MaybePut(path, contents, callback): + if contents: + yield gen.Task(merged_store.Put, path, contents) + logging.info('Wrote %d bytes to %s' % (len(contents), path)) + callback() + + + yield [gen.Task(_MaybePut, req_file_path, req_contents), + gen.Task(_MaybePut, dev_file_path, dev_contents), + gen.Task(_MaybePut, trace_file_path, trace_contents)] + + last_day_written = day_stats.day + + callback(last_day_written) + return + + +@gen.engine +def GetMergedLogsFileList(merged_store, logs_paths, marker, callback): + """Fetch the list of file names from S3.""" + registry_file = logs_paths.ProcessedRegistryPath() + def _WantFile(filename): + if filename == registry_file: + return False + instance = logs_paths.MergedLogPathToInstance(filename) + if instance is None: + logging.error('Could not extract instance from file name %s' % filename) + return False + return not options.options.ec2_only or logs_util.IsEC2Instance(instance) + + base_path = logs_paths.MergedDirectory() + marker = os.path.join(base_path, marker) if marker is not None else None + file_list = yield gen.Task(store_utils.ListAllKeys, merged_store, prefix=base_path, marker=marker) + files = [f for f in file_list if _WantFile(f)] + files.sort() + + logging.info('found %d merged log files, analyzing %d' % (len(file_list), len(files))) + callback(files) + + +@gen.engine +def RunOnce(client, job, callback): + """Get list of files and call processing function.""" + logs_paths = logs_util.ServerLogsPaths('viewfinder', 'full') + merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET) + + start_date = options.options.start_date + if options.options.smart_scan: + # Search for successful full-scan run in the last week. + last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day') + if last_run is None: + logging.info('No previous successful scan found, rerun with --start_date') + callback(None) + return + + last_run_start = last_run['start_time'] + if util.HoursSince(last_run_start) < options.options.hours_between_runs: + logging.info('Last successful run started at %s, less than %d hours ago; skipping.' % + (time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs)) + callback(None) + return + + last_day = last_run['stats.last_day'] + # Set scan_start to start of previous run - 1d. The extra 1d is in case some logs were pushed to S3 late. + # This really recomputes two days (the last day that was successfully processed and the one prior). + start_time = util.ISO8601ToUTCTimestamp(last_day, hour=12) - constants.SECONDS_PER_DAY + start_date = util.TimestampUTCToISO8601(start_time) + logging.info('Last successful analyze_logs run (%s) scanned up to %s, setting analysis start date to %s' % + (time.asctime(time.localtime(last_run_start)), last_day, start_date)) + + # Fetch list of merged logs. + files = yield gen.Task(GetMergedLogsFileList, merged_store, logs_paths, start_date) + last_day = yield gen.Task(ProcessFiles, merged_store, logs_paths, files) + callback(last_day) + return + + +@gen.engine +def _Start(callback): + """Grab a lock on job:analyze_logs and call RunOnce. If we get a return value, write it to the job summary.""" + client = db_client.DBClient.Instance() + job = Job(client, 'analyze_logs') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + result = None + job.Start() + try: + result = yield gen.Task(RunOnce, client, job) + except: + # Failure: log run summary with trace. + typ, val, tb = sys.exc_info() + msg = ''.join(traceback.format_exception(typ, val, tb)) + logging.info('Registering failed run with message: %s' % msg) + yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) + else: + if result is not None and not options.options.dry_run: + # Successful run with data processed and not in dry-run mode: write run summary. + stats = DotDict() + stats['last_day'] = result + logging.info('Registering successful run with stats: %r' % stats) + yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/generate_pdf_report.py b/backend/logs/generate_pdf_report.py new file mode 100644 index 0000000..24f0409 --- /dev/null +++ b/backend/logs/generate_pdf_report.py @@ -0,0 +1,321 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Generate graphs from metrics data and output as pdf. + +The PDF files are written to S3 and links send by email. + +Simply run with: +python -m viewfinder.backend.logs.generate_pdf_report --devbox + +Options: +- require_lock: default=True: grab a lock on 'generate_pdf_reports' before running +- analysis_intervals_days: default=14,90: generate one file for each interval +- upload_to_s3: default=True: upload PDF files to S3 +- s3_dest: default='pdf_reports': directory in S3 to write to (inside bucket 'serverdata') +- local_working_dir: default='/tmp/': write pdf files to this local dir (they are not deleted) +- send_email: default=True: send an email report. If false, uses the LoggingEmailManager +- email: default=marketing@emailscrubbed.com: email recipient +- s3_url_expiration_days: default=14: time to live for the generated S3 urls. +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import json +import logging +import os +import re +import sys +import time +import traceback + +from collections import Counter, defaultdict +from datetime import datetime +from tornado import gen, options +from viewfinder.backend.base import constants, main, retry, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, metric +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util +from viewfinder.backend.services.email_mgr import EmailManager, LoggingEmailManager, SendGridEmailManager +from viewfinder.backend.storage.object_store import ObjectStore + +import matplotlib.pyplot as plt +from matplotlib import dates as mdates +from matplotlib.backends.backend_pdf import PdfPages + +options.define('require_lock', default=True, help='Acquire job lock on "generate_pdf_reports" before running') +options.define('analysis_intervals_days', default=[14, 90], help='Intervals to analyze, in days') +options.define('upload_to_s3', default=True, help='Upload generated files to S3') +options.define('s3_dest', default='pdf_reports', help='S3 directory to write to (in serverdata bucket)') +options.define('local_working_dir', default='/tmp/', help='Local directory to write generated pdf files to') +options.define('send_email', default=True, help='Email links to reports') +options.define('email', default='analytics-reports@emailscrubbed.com', help='Email recipient') +options.define('s3_url_expiration_days', default=14, help='Expiration time in days for S3 URLs') + +# Retry policy for uploading files to S3. +kS3UploadRetryPolicy = retry.RetryPolicy(max_tries=5, timeout=300, + min_delay=1, max_delay=30, + check_exception=retry.RetryPolicy.AlwaysRetryOnException) + +# Metrics to sum up into a new one. +kSummedMetrics = [ (re.compile(r'itunes\.downloads.*'), r'itunes.downloads'), + (re.compile(r'itunes\.updates.*'), r'itunes.updates'), + ] + +# Metric selection. +kFilteredMetrics = [ r'itunes\.updates', + r'itunes\.download', + r'db\.table\.count\.(Comment|Photo|User)$', + r'active_users\.requests_(all|share|post)\.(1d|7d|30d)', + ] + +# Metrics to draw on the same graph, associated title and legend. +kPlotAggregates = { + r'(active_users\.requests_all)\.(1d|7d|30d)': { + 'title_rep': r'Active users (all requests)', + 'legend_rep': r'\2', + }, + r'(active_users\.requests_post)\.(1d|7d|30d)': { + 'title_rep': r'Active users posting comments', + 'legend_rep': r'\2', + }, + r'(active_users\.requests_share)\.(1d|7d|30d)': { + 'title_rep': r'Active users sharing photos', + 'legend_rep': r'\2', + }, + r'db\.table\.count\.(Comment|Photo|User)': { + 'title_rep': r'Total \1s', + 'legend_rep': None, + }, + r'itunes\.(downloads|update)': { + 'title_rep': r'Daily iTunes \1', + 'legend_rep': None, + }, +} + +def SerializeMetrics(metrics): + def _SkipMetric(name): + for regex in kFilteredMetrics: + res = re.match(regex, k) + if res is not None: + return False + return True + + def _AggregateMetric(running_sum, metric_name): + """Given a metric name, determine whether we sum it into a different metric name or not. + Returns whether the original metric needs to be processed. + """ + keep = True + for regex, replacement, in kSummedMetrics: + res = regex.sub(replacement, metric_name) + if res != metric_name: + keep = False + if not _SkipMetric(res): + running_sum[res] += v + return keep + + data = defaultdict(list) + prev_metrics = {} + seen_vars = set() + for m in metrics: + running_sum = Counter() + timestamp = m.timestamp + payload = DotDict(json.loads(m.payload)).flatten() + for k, v in payload.iteritems(): + keep_original = _AggregateMetric(running_sum, k) + if keep_original and not _SkipMetric(k): + running_sum[k] += v + for k, v in running_sum.iteritems(): + data[k].append((timestamp, v)) + + return data + +@gen.engine +def ProcessOneInterval(client, num_days, callback): + end_time = time.time() + start_time = time.time() - constants.SECONDS_PER_DAY * num_days + + selected_interval = metric.LOGS_INTERVALS[-1] + group_key = metric.Metric.EncodeGroupKey(metric.LOGS_STATS_NAME, selected_interval) + logging.info('Query performance counters %s, range: %s - %s, resolution: %s' + % (group_key, time.ctime(start_time), time.ctime(end_time), selected_interval.name)) + + metrics = list() + start_key = None + while True: + new_metrics = yield gen.Task(metric.Metric.QueryTimespan, client, group_key, + start_time, end_time, excl_start_key=start_key) + if len(new_metrics) > 0: + metrics.extend(new_metrics) + start_key = metrics[-1].GetKey() + else: + break + + data = SerializeMetrics(metrics) + + def _DetermineTitle(metric_name): + for regex, props in kPlotAggregates.iteritems(): + if not re.match(regex, metric_name): + continue + tres = re.sub(regex, props['title_rep'], metric_name) + legend_rep = props.get('legend_rep', None) + if not legend_rep: + return (tres, None) + else: + vres = re.sub(regex, legend_rep, metric_name) + return (tres, vres) + return (metric_name, metric_name) + + def _SaveFig(legend_data): + logging.info('Drawing with legend_data=%r' % legend_data) + if legend_data: + # Shrink the figure vertically. + box = plt.gca().get_position() + plt.gca().set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8]) + + # Put a legend below current axis + plt.legend(legend_data, loc='upper center', bbox_to_anchor=(0.5, -0.20), + fancybox=True, shadow=True, ncol=5) + elif plt.legend(): + plt.legend().set_visible(False) + + # Write to pdf as a new page. + plt.savefig(pp, format='pdf') + + # Clear all. + plt.clf() + plt.cla() + + # PdfPages overwrites any existing files. Should unlink fail, we'll let the exception surface. + filename = '%dd-viewfinder-report.%s.pdf' % (num_days, util.NowUTCToISO8601()) + pp = PdfPages(os.path.join(options.options.local_working_dir, filename)) + last_entry = None + legend_strings = [] + for k in sorted(data.keys()): + timestamps = [] + y_axis = [] + for a, b in data[k]: + dt = datetime.utcfromtimestamp(a) + dt = dt.replace(hour=0) + timestamps.append(dt) + y_axis.append(b) + + x_axis = mdates.date2num(timestamps) + + title, label = _DetermineTitle(k) + + if last_entry is not None and last_entry != title: + # Different data set: draw figure, write to pdf and clear everything. + _SaveFig(legend_strings) + legend_strings = [] + + last_entry = title + if label: + legend_strings.append(label) + + # autofmt_xdate sets the formatter and locator to AutoDate*. It seems smart enough. + # plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m/%d/%Y')) + # plt.gca().xaxis.set_major_locator(mdates.DayLocator()) + plt.title(title) + plt.grid(True) + + # Plot data. + plt.plot_date(x_axis, y_axis, '-') + plt.gcf().autofmt_xdate() + + _SaveFig(legend_strings) + pp.close() + + callback(filename) + +@gen.engine +def UploadFiles(object_store, filenames, callback): + for f in filenames: + local_file = os.path.join(options.options.local_working_dir, f) + contents = open(local_file, 'r').read() + + remote_file = os.path.join(options.options.s3_dest, f) + + # Assume 1MB/s transfer speed. If we don't have that good a connection, we really shouldn't be uploading big files. + timeout = max(20.0, len(contents) / 1024 * 1024) + yield gen.Task(retry.CallWithRetryAsync, kS3UploadRetryPolicy, + object_store.Put, remote_file, contents, request_timeout=timeout) + logging.info('Uploaded %d bytes to S3 file %s' % (len(contents), remote_file)) + + callback() + + +@gen.engine +def SendEmail(title, text, callback): + args = { + 'from': 'analytics-reports@emailscrubbed.com', + 'fromname': 'Viewfinder reports', + 'to': options.options.email, + 'subject': title, + 'text': text + } + yield gen.Task(EmailManager.Instance().SendEmail, description=title, **args) + callback() + + +@gen.engine +def SendReport(object_store, filename_dict, callback): + text = 'Viewfinder statistics report:\n' + text += '(URLs expire after %d days)\n\n' % options.options.s3_url_expiration_days + for days in sorted(filename_dict.keys()): + filename = filename_dict[days] + remote_file = os.path.join(options.options.s3_dest, filename) + url = object_store.GenerateUrl(remote_file, + expires_in=constants.SECONDS_PER_DAY * options.options.s3_url_expiration_days, + content_type='application/pdf') + text += 'Past %d days: %s\n\n' % (days, url) + + title = 'Viewfinder statistics report: %s' % util.NowUTCToISO8601() + yield gen.Task(SendEmail, title, text) + callback() + + +@gen.engine +def RunOnce(client, callback): + object_store = ObjectStore.GetInstance(ObjectStore.SERVER_DATA) + filenames = {} + + for num_days in options.options.analysis_intervals_days: + filename = yield gen.Task(ProcessOneInterval, client, num_days) + filenames[num_days] = filename + + yield gen.Task(UploadFiles, object_store, filenames.values()) + yield gen.Task(SendReport, object_store, filenames) + callback() + + +@gen.engine +def Start(callback): + client = db_client.DBClient.Instance() + + job = Job(client, 'generate_pdf_reports') + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + if options.options.send_email: + # When running on devbox, this prompts for the passphrase. Skip if not sending email. + EmailManager.SetInstance(SendGridEmailManager()) + else: + EmailManager.SetInstance(LoggingEmailManager()) + + # We never call job.Start() since we don't want a summary status written to the DB, just the lock. + try: + yield gen.Task(RunOnce, client) + except: + logging.error(traceback.format_exc()) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + +if __name__ == '__main__': + sys.exit(main.InitAndRun(Start)) diff --git a/backend/logs/get_client_logs.py b/backend/logs/get_client_logs.py new file mode 100644 index 0000000..db472a9 --- /dev/null +++ b/backend/logs/get_client_logs.py @@ -0,0 +1,276 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved. + +"""Merge user analytics logs into per-day/per-user files. + +For each user, we look at files not yet processed (there is a per-user registry in serverdata/). +For each individual analytic entry, we determine which day to write it to based on the entry timestamp and wrap +it inside a small dict containing device_id and version (from the raw filename). + +Usage: +# Process analytics logs for all users +python -m viewfinder.backend.logs.get_client_logs + +# Search for new logs starting on December 1st 2012 (S3 ListKeys prefix). +python -m viewfinder.backend.logs.get_client_logs --start_date=2012-12-01 + +Other flags: +-dry_run: default=True: do everything, but do not write processed logs files to S3 or update registry. +-require_lock: default=True: hold the job:client_logs lock during processing. +-user: default=None: process a single user. If None, process all users. +-start_user: default=None: start scanning from this user id (lexicographically, not numerically) + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import json +import logging +import os +import sys + +from tornado import gen, options +from viewfinder.backend.base import main, retry, util +from viewfinder.backend.db import db_client +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import log_merger, logs_util +from viewfinder.backend.storage import store_utils +from viewfinder.backend.storage.object_store import ObjectStore + +options.define('start_date', default=None, help='Start date (filename start key)') +options.define('dry_run', default=True, help='Do not write processed logs to S3 or update registry') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:client_logs lock before running. Exit if acquire fails.') +options.define('user', default=None, help='Process a single user') +options.define('start_user', default=None, + help='start scanning from this user id (lexicographically, not numerically)') +options.define('max_users', default=None, type=int, + help='maximum number of users to examine') +options.define('process_analytics', default=True, type=bool, + help='process user analytics files') +options.define('process_crashes', default=True, type=bool, + help='process user crash files') + +# Retry policy for uploading files to S3 (merge logs and registry). +kS3UploadRetryPolicy = retry.RetryPolicy(max_tries=5, timeout=300, + min_delay=1, max_delay=30, + check_exception=retry.RetryPolicy.AlwaysRetryOnException) + +@gen.engine +def HandleOneUser(client_store, user_id, callback): + """Process client logs for a single user.""" + logs_paths = logs_util.UserAnalyticsLogsPaths(user_id) + + # List all files for this user. + base_path = logs_paths.RawDirectory() + marker = os.path.join(base_path, options.options.start_date) if options.options.start_date is not None else None + files = yield gen.Task(store_utils.ListAllKeys, client_store, prefix=base_path, marker=marker) + analytics_files = [] + crash_files = [] + for f in sorted(files): + if f.endswith('.analytics.gz'): + analytics_files.append(f) + elif f.endswith('.crash') or f.endswith('.crash.gz'): + crash_files.append(f) + + if analytics_files and options.options.process_analytics: + yield gen.Task(HandleAnalytics, client_store, user_id, logs_paths, analytics_files) + if crash_files and options.options.process_crashes: + yield gen.Task(HandleCrashes, client_store, user_id, crash_files) + + callback() + +@gen.engine +def HandleCrashes(client_store, user_id, raw_files, callback): + logs_paths = logs_util.UserCrashLogsPaths(user_id) + raw_store = ObjectStore.GetInstance(logs_paths.SOURCE_LOGS_BUCKET) + merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET) + + # List all processed + base_path = logs_paths.MergedDirectory() + existing_files = yield gen.Task(store_utils.ListAllKeys, merged_store, prefix=base_path, marker=None) + done_files = set() + for e in existing_files: + parsed = logs_paths.ParseMergedLogPath(e) + if parsed: + done_files.add(parsed) + + to_copy = [] + for f in raw_files: + parsed = logs_paths.ParseRawLogPath(f) + if not parsed or parsed in done_files: + continue + to_copy.append(parsed) + + if to_copy: + logging.info('User %s: %d crash files' % (user_id, len(to_copy))) + + if options.options.dry_run: + callback() + return + + @gen.engine + def _CopyFile(source_parsed, callback): + user, date, fname = source_parsed + src_file = os.path.join(logs_paths.RawDirectory(), date, fname) + dst_file = os.path.join(logs_paths.MergedDirectory(), date, fname) + contents = yield gen.Task(raw_store.Get, src_file) + yield gen.Task(merged_store.Put, dst_file, contents) + callback() + + yield [gen.Task(_CopyFile, st) for st in to_copy] + callback() + +@gen.engine +def HandleAnalytics(client_store, user_id, logs_paths, raw_files, callback): + s3_base = logs_paths.MergedDirectory() + raw_store = ObjectStore.GetInstance(logs_paths.SOURCE_LOGS_BUCKET) + merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET) + + # Fetch user's repository of processed files. + processed_files = yield gen.Task(logs_util.GetRegistry, merged_store, logs_paths.ProcessedRegistryPath()) + if processed_files is None: + # None means: registry does not exist. All other errors throw exceptions. + processed_files = [] + + # Compute list of raw files to process (and sort by filename -> sort by date). + files_set = set(raw_files) + processed_set = set(processed_files) + missing_files = list(files_set.difference(processed_set)) + missing_files.sort() + + if len(missing_files) == 0: + callback() + return + + # Dict of 'day' to LocalLogMerge for that day and user. + day_mergers = {} + + finished_files = [] + + for i in missing_files: + res = logs_paths.ParseRawLogPath(i) + assert res is not None, 'Problem parsing file %s' % i + assert res[0] == 'analytics', 'Problem parsing file %s' % i + assert res[1] == user_id, 'Problem parsing file %s' % i + device_id = res[2] + version = res[3] + + # We wrap the entire file processing into a single try statement. Any problems with is and we discard all + # entries in the log mergers and don't add it to the list of processed files. + try: + # GetFileContents automatically gunzips files based on extension. + contents = yield gen.Task(store_utils.GetFileContents, client_store, i) + # Clients don't know when the file is closed so will not write a terminating bracket. Make it conditional just + # in case they do one day. + if contents.startswith('[') and not contents.endswith(']'): + contents += ']' + parsed = json.loads(contents) + for p in parsed: + if not 'timestamp': + logging.warning('Analytics entry without timestamp: %r' % p) + continue + day = util.TimestampUTCToISO8601(p['timestamp']) + container = {'device_id': device_id, 'payload': p} + # Some (old?) user logs don't have a version embedded in the filename. + if version is not None: + container['version'] = version + + merger = day_mergers.get(day, None) + if merger is None: + merger = log_merger.LocalLogMerge(merged_store, [day, user_id], s3_base) + yield gen.Task(merger.FetchExistingFromS3) + day_mergers[day] = merger + + # Each line is individually json encoded, but we do not terminate the file as we don't know when we'll + # be adding more data to it. + merger.Append(json.dumps(container)) + except Exception as e: + # We don't currently surface the error and interrupt the entire job as we want to find any badly-formatter files. + # TODO(marc): this won't be very visible, we should create a list of bad files we can examine later. + logging.warning('Problem processing %s: %r' % (i, e)) + for merger in day_mergers.values(): + merger.DiscardBuffer() + continue + + # We put log flushing after the try statement. We do want to catch this. + for merger in day_mergers.values(): + merger.FlushBuffer() + finished_files.append(i) + + + # Close all mergers for this user + tasks = [] + for day, merger in day_mergers.iteritems(): + merger.Close() + tasks.append(gen.Task(merger.Upload)) + + if not options.options.dry_run: + try: + yield tasks + except Exception as e: + # Errors in the Put are unrecoverable. We can't upload the registry after a failed merged log upload. + # TODO(marc): provide a way to mark a given day's merged logs as bad and force recompute. + logging.error('Error uploading file(s) to S3 for user %s: %r' % (user_id, e)) + + # Commit the registry file. + processed_files.extend(finished_files) + processed_files.sort() + if not options.options.dry_run: + yield gen.Task(retry.CallWithRetryAsync, kS3UploadRetryPolicy, + logs_util.WriteRegistry, merged_store, logs_paths.ProcessedRegistryPath(), processed_files) + + # Now cleanup the merger objects. + for merger in day_mergers.values(): + merger.Cleanup() + + callback() + +@gen.engine +def RunOnce(callback): + """Get list of files and call processing function.""" + dry_run = options.options.dry_run + client_store = ObjectStore.GetInstance(logs_util.UserAnalyticsLogsPaths.SOURCE_LOGS_BUCKET) + + if options.options.user: + users = [options.options.user] + else: + users = yield gen.Task(logs_util.ListClientLogUsers, client_store) + + examined = 0 + for u in users: + # Running all users in parallel can get us to exceed the open FD limit. + if options.options.start_user is not None and u < options.options.start_user: + continue + if options.options.max_users is not None and examined > options.options.max_users: + break + examined += 1 + yield gen.Task(HandleOneUser, client_store, u) + + if dry_run: + logging.warning('dry_run=True: will not upload processed logs files or update registry') + + callback() + +@gen.engine +def _Start(callback): + """Grab the job lock and call RunOnce if acquired. We do not write a job summary as we currently do not use it.""" + client = db_client.DBClient.Instance() + job = Job(client, 'client_logs') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + try: + yield gen.Task(RunOnce) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/get_server_logs.py b/backend/logs/get_server_logs.py new file mode 100644 index 0000000..1c0db7e --- /dev/null +++ b/backend/logs/get_server_logs.py @@ -0,0 +1,250 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Merge server logs from S3 into daily directories, with one file per instance. + +Does the following: +1) fetch list of log files from S3://serverlog-viewfinder-co/viewfinder/full/ + fetch list of processed files from file S3://serverlog-viewfinder-co/processed_logs/viewfinder/full/PROCESSED +2) compute list of raw log files that need to be processed +3) iterate over all log files and write filtered entries (by module) to a per-day/per-instance file + if a file exists in S3 for that day/instance, fetch it first to merge +4) when all files have been processed, upload all working files to S3 and update the list of processed files. + +The resulting merged files are in: S3://serverlog-viewfinder-co/processed_logs/viewfinder/full/YYYY-MM-DD/ +All entries in a given merged file are guaranteed to be for that date only. A single file is created per instance. + +Only log entries generated by one of the following modules are kept: +['identity', 'operation', 'service', 'user_op_manager', 'web'] + +When processing a raw log file, entries are not directly written to the new merged file(s) but instead kept in a +per-merged-file buffer which is only written our once the raw log has been fully processed. At that point, the +filename of the raw log file is also added to the "processed" list. + +When all log files to be processed have been, upload all files to S3 and update the registry with the list of +files successfilly processed. + +Usage: +# Process all new server logs since last fetch. +python -m viewfinder.backend.logs.get_server_logs + +# Search for new logs starting on December 1st 2012 (S3 ListKeys prefix). +python -m viewfinder.backend.logs.get_server_logs --start_date=2012-12-01 + +Other flags: +-dry_run: default=True: do everything, but do not write processed logs files to S3 or update registry. +-ec2_only: default=True: only analyze logs from AWS instances. +-require_lock: default=True: hold the job:merge_logs lock during processing. +-max_files_to_process: default=None: process at most this many raw log files. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import cStringIO +import logging +import os +import re +import sys + +from tornado import gen, options +from viewfinder.backend.base import main, retry +from viewfinder.backend.db import db_client +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util, log_merger +from viewfinder.backend.storage import store_utils +from viewfinder.backend.storage.object_store import ObjectStore + +options.define('start_date', default=None, help='Start date (filename start key)') +options.define('dry_run', default=True, help='Do not write processed logs to S3 or update registry') +options.define('ec2_only', default=True, help='AWS instances only') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:merge_logs lock before running. Exit if acquire fails.') +options.define('max_files_to_process', type=int, default=None, help='Maximum number of files to process.') + +# Retry policy for uploading files to S3 (merge logs and registry). +kS3UploadRetryPolicy = retry.RetryPolicy(max_tries=5, timeout=300, + min_delay=1, max_delay=30, + check_exception=retry.RetryPolicy.AlwaysRetryOnException) + +@gen.engine +def ProcessFiles(logs_store, merged_store, logs_paths, filenames, dry_run, callback): + """Process each file contained in 'filenames'. Returns a list of successfully processed file names.""" + processed_files = [] + # Dict of (instance, date) -> LocalLogMerge + day_instance_logs = {} + s3_base = logs_paths.MergedDirectory() + + @gen.engine + def _AddEntry(instance, entry, callback): + """Given a single log entry, validate it, find (or create) the corresponding LocalLogMerge and add it.""" + if not entry: + callback() + return + parsed = logs_util.ParseLogLine(entry) + if not parsed: + callback() + return + day, _, _, _ = parsed + + # We can't use setdefault here, as it would create a new LocalLogMerge instance. + day_log = None + if (instance, day) not in day_instance_logs: + # S3 filenames will be: /s3_base/day/instance + day_instance_logs[(instance, day)] = log_merger.LocalLogMerge(merged_store, [day, instance], s3_base) + day_log = day_instance_logs[(instance, day)] + yield gen.Task(day_log.FetchExistingFromS3) + day_log = day_instance_logs[(instance, day)] + day_log.Append(entry) + callback() + + @gen.engine + def _ProcessOneFile(instance, contents, callback): + """Iterate over a raw log file. Any line that does not start with a date in the form YYYY-MM-DD is considered + part of a multi-line entry. + TODO(marc): this will break horribly if we ever have multiple processes/threads logging to the same file. + """ + buf = cStringIO.StringIO(contents) + buf.seek(0) + entry = '' + while True: + line = buf.readline() + if not line: + yield gen.Task(_AddEntry, instance, entry) + break + if line.startswith((' ', '\t')) or logs_util.ParseLogLine(line) is None: + # This is a followup to a multi-line entry. + entry += ' ' + line.strip() + else: + # New entry: write the previous one. + yield gen.Task(_AddEntry, instance, entry) + entry = line.strip() + buf.close() + callback() + + for filename in filenames: + instance = logs_paths.RawLogPathToInstance(filename) + assert instance + contents = '' + try: + contents = yield gen.Task(logs_store.Get, filename) + except Exception as e: + logging.error('Error fetching file %s: %r' % (filename, e)) + continue + + logging.info('processing %d bytes from raw log %s' % (len(contents), filename)) + yield gen.Task(_ProcessOneFile, instance, contents) + processed_files.append(filename) + # We need to flush all DayInstance logs as we've marked the file as successfully processed. + for log in day_instance_logs.values(): + log.FlushBuffer() + + # Close all LocalMergeLog objects and upload to S3. + for instance_day, log in day_instance_logs.iteritems(): + log.Close() + if not dry_run: + try: + yield gen.Task(log.Upload) + except Exception as e: + # Errors in the Put are unrecoverable. We can't upload the registry after a failed merged log upload. + # TODO(marc): provide a way to mark a given day's merged logs as bad and force recompute. + logging.error('Error uploading file to S3 for %r: %r' % (instance_day, e)) + log.Cleanup() + callback(processed_files) + + +@gen.engine +def GetRawLogsFileList(logs_store, logs_paths, marker, callback): + """Fetch the list of file names from S3.""" + def _WantFile(filename): + instance = logs_paths.RawLogPathToInstance(filename) + if instance is None: + logging.error('Could not extract instance from file name %s' % filename) + return False + return not options.options.ec2_only or logs_util.IsEC2Instance(instance) + + base_path = logs_paths.RawDirectory() + marker = os.path.join(base_path, marker) if marker is not None else None + file_list = yield gen.Task(store_utils.ListAllKeys, logs_store, prefix=base_path, marker=marker) + files = [f for f in file_list if _WantFile(f)] + + logging.info('found %d total raw log files, %d important ones' % (len(file_list), len(files))) + callback(files) + + +@gen.engine +def RunOnce(callback): + """Get list of files and call processing function.""" + dry_run = options.options.dry_run + + logs_paths = logs_util.ServerLogsPaths('viewfinder', 'full') + + if dry_run: + logging.warning('dry_run=True: will not upload processed logs files or update registry') + + logs_store = ObjectStore.GetInstance(logs_paths.SOURCE_LOGS_BUCKET) + merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET) + + # Fetch list of raw logs files. + files = yield gen.Task(GetRawLogsFileList, logs_store, logs_paths, options.options.start_date) + + # Fetch list of processed logs files from registry. + processed_files = yield gen.Task(logs_util.GetRegistry, merged_store, logs_paths.ProcessedRegistryPath()) + if processed_files is None: + # None means: registry does not exist. All other errors throw exceptions. + processed_files = [] + + # Compute list of raw files to process (and sort by filename -> sort by date). + files_set = set(files) + processed_set = set(processed_files) + missing_files = list(files_set.difference(processed_set)) + missing_files.sort() + + to_process = missing_files + if options.options.max_files_to_process is not None: + to_process = missing_files[0:options.options.max_files_to_process] + + logging.info('found %d raw files and %d processed files, %d missing. Will process %d.' % + (len(files), len(processed_files), len(missing_files), len(to_process))) + if len(missing_files) == 0: + logging.info('No raw logs files to process.') + callback() + return + + merged_files = yield gen.Task(ProcessFiles, logs_store, merged_store, logs_paths, to_process, dry_run) + logging.info('found %d raw files and %d processed files, %d missing, successfully processed %d' % + (len(files), len(processed_files), len(missing_files), len(merged_files))) + + # Add processed files to registry and write to S3. + # TODO(marc): any failure in merged log upload or registry upload will cause us to get out of sync. To fix this, + # we should also have a list of properly applied processed logs. + processed_files.extend(merged_files) + processed_files.sort() + if not dry_run: + yield gen.Task(retry.CallWithRetryAsync, kS3UploadRetryPolicy, + logs_util.WriteRegistry, merged_store, logs_paths.ProcessedRegistryPath(), processed_files) + + callback() + +@gen.engine +def _Start(callback): + """Grab the job lock and call RunOnce if acquired. We do not write a job summary as we currently do not use it.""" + client = db_client.DBClient.Instance() + job = Job(client, 'merge_logs') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + try: + yield gen.Task(RunOnce) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/get_table_sizes.py b/backend/logs/get_table_sizes.py new file mode 100644 index 0000000..4fa1d9e --- /dev/null +++ b/backend/logs/get_table_sizes.py @@ -0,0 +1,71 @@ +# Copyright 2013 Viewfinder Inc. All Rights Reserved +"""Fetch dynamodb table sizes and counts and store in logs.daily metric. + +Run with: +$ python -m viewfinder.backend.logs.get_table_sizes --dry_run=False + +Options: +- dry_run: default=True: run in dry-run mode (don't write to dynamodb) +- require_lock: default=True: grab the job:itunes_trends lock for the duration of the job. +""" + +import json +import logging +import sys +import time + +from tornado import gen, options +from viewfinder.backend.base import main, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, metric, vf_schema +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util + +options.define('dry_run', default=True, help='Print output only, do not write to metrics table.') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:itunes_trends lock before running. Exit if acquire fails.') + + +@gen.engine +def RunOnce(client, callback): + today = util.NowUTCToISO8601() + logging.info('getting table sizes for %s' % today) + + results = yield gen.Task(vf_schema.SCHEMA.VerifyOrCreate, client, verify_only=True) + stats = DotDict() + for r in sorted(results): + name = r[0] + props = r[1] + stats['db.table.count.%s' % name] = props.count + stats['db.table.size.%s' % name] = props.size_bytes + + # Replace the entire 'db.table' prefix in previous metrics. + hms = logs_util.kDailyMetricsTimeByLogType['dynamodb_stats'] + yield gen.Task(logs_util.UpdateMetrics, client, {today: stats}, prefix_to_erase='db.table', + dry_run=options.options.dry_run, hms_tuple=hms) + callback() + + +@gen.engine +def _Start(callback): + """Grab a lock on job:table_sizes and call RunOnce. We never write a job summary.""" + client = db_client.DBClient.Instance() + job = Job(client, 'table_sizes') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + try: + yield gen.Task(RunOnce, client) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/itunes_trends.py b/backend/logs/itunes_trends.py new file mode 100644 index 0000000..2bd4cbb --- /dev/null +++ b/backend/logs/itunes_trends.py @@ -0,0 +1,287 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved +"""Interface to the iTunes Connect sales and trend. + +Fetches daily download stats from iTunes Connect and saves them to the metrics table. +Specify the apple user to log in with (eg: --user=marc to use marc@emailscrubbed.com). We get the user's +apple password from the secret 'itunes_connect_${user}'. +Default user is 'itunes_viewer', a special user with "sales" data access only. + +Run with: +$ python -m viewfinder.backend.logs.itunes_trends --start_date=2013-01-20 + +Appropriate for cron: Detect start date, update metrics table. Don't run if last run was less than 6h ago. +$ python -m viewfinder.backend.logs.itunes_trends --dry_run=False --smart_scan=True --hours_between_runs=6 + + ExceptionOptions: +- user: default=itunes_viewer: apple user. We expand this to ${user}@emailscrubbed.com +- vendor_id: default=: the vendor ID, from the iTunes Connect dashboard. +- dry_run: default=True: display stats only, do not update Metrics table or write job summary. +- start_date: default=None: look up stats from that date until yesterday. format: YYYY-MM-DD. +- smart_scan: default=False: determine the start date from previous successful run. +- require_lock: default=True: grab the job:itunes_trends lock for the duration of the job. +- hours_between_runs: default=0: don't run if the last successful run was less than this many hours ago. + +""" + +import gzip +import cStringIO +import getpass +import json +import logging +import os +import re +import sys +import time +import traceback +import urllib +import urllib2 + +from tornado import gen, options +from urlparse import urljoin +from viewfinder.backend.base import constants, main, secrets, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import db_client, metric +from viewfinder.backend.db.job import Job +from viewfinder.backend.logs import logs_util +from viewfinder.backend.services import itunes_trends_codes +from viewfinder.backend.storage.object_store import ObjectStore + +options.define('user', default='itunes_viewer', help='User for iTunesConnect. Will expand to user@emailscrubbed.com and ' + 'lookup the itunes password in secrets/itunes_connect_${user}') +# vendor_id comes from the iTunes Connect dashboard. +options.define('vendor_id', default='85575078', help='iTunes vendor ID') + +options.define('dry_run', default=True, help='Print output only, do not write to metrics table.') +options.define('start_date', default=None, help='Lookup stats from date onwards (YYYY-MM-DD)') +options.define('smart_scan', type=bool, default=False, + help='determine start_date from previous successful runs. Overrides start_date.') +options.define('require_lock', type=bool, default=True, + help='attempt to grab the job:itunes_trends lock before running. Exit if acquire fails.') +options.define('hours_between_runs', type=int, default=0, + help='minimum time since start of last successful run (with dry_run=False)') +options.define('download_from_s3', type=bool, default=False, + help='Fetch raw gzipped files from S3 (if false, fetch from iTunesConnect)') + +kITCBaseURL = 'https://reportingitc.apple.com/autoingestion.tft' +kS3Bucket = ObjectStore.SERVER_DATA +kS3Base = 'itunes-trends/' + +class ITunesTrends(object): + def __init__(self, apple_id, password, vendor_id, html_retries=3): + self._apple_id = apple_id + self._password = password + self._vendor_id = vendor_id + self._html_retries=3 + + self._form_fields = None + self._available_days = None + self._available_weeks = None + + self._object_store = ObjectStore.GetInstance(kS3Bucket) + + def _Fetch(self, url, data=None): + """Attempt to fetch 'url' with optional 'data'. We retry self._retry times, regardless of the error.""" + retries = 0 + while True: + logging.info('fetching (%d) %s' % (retries, url)) + request = urllib2.Request(url, data) + handle = urllib2.urlopen(request) + logging.info('fetch reply headers: %s' % handle.info()) + return handle.read() + try: + pass + except Exception: + if retries >= self._html_retries: + raise + + time.sleep(2**retries) + retries += 1 + + + @gen.engine + def FetchOneDay(self, day, callback): + """Fetch a single day's worth of data. Exception could be due to http errors, unavailable date, or failed parsing. + TODO(marc): handle these cases separately. + """ + s3_filename = os.path.join(kS3Base, '%s.gz' % day) + + def DownloadFromiTunes(): + # We use our own date format in the entire tool. Only now do we convert to iTuneConnect's YYYYMMDD. + y, m, d = day.split('-') + itunes_date = '%s%s%s' % (y, m, d) + data = urllib.urlencode({'USERNAME': self._apple_id, + 'PASSWORD': self._password, + 'VNDNUMBER': self._vendor_id, + 'TYPEOFREPORT': 'Sales', + 'DATETYPE': 'Daily', + 'REPORTTYPE': 'Summary', + 'REPORTDATE': itunes_date }) + buf = self._Fetch(kITCBaseURL, data) + return buf + + def ParseContents(contents): + result = DotDict() + skipped_lines = [] + for line in contents.splitlines(): + tokens = line.split('\t') + if tokens[0] == 'Provider': + # Skip header line. + skipped_lines.append(line) + continue + # Replace dots with underscores as we'll be using the version in a DotDict. + version = tokens[5].replace('.', '_') + if not version or version == ' ': + # subscriptions do not have a version, use 'all'. + version = 'all' + type_id = tokens[6] + # Use the type id if we don't have a name for it. + type_name = itunes_trends_codes.PRODUCT_TYPE_IDENTIFIER.get(type_id, type_id) + units = int(tokens[7]) + # Ignore proceeds, it does not reflect in-app purchases. + store = tokens[12] + result['itunes.%s.%s.%s' % (type_name, version, store)] = units + assert len(skipped_lines) <= 1, 'Skipped too many lines: %r' % skipped_lines + return result + + # Failures in any of Get/Download/Put will interrupt this day's processing. + if options.options.download_from_s3: + logging.info('S3 get %s' % s3_filename) + buf = yield gen.Task(self._object_store.Get, s3_filename) + else: + buf = DownloadFromiTunes() + logging.info('S3 put %s' % s3_filename) + yield gen.Task(self._object_store.Put, s3_filename, buf) + + iobuffer = cStringIO.StringIO(buf) + gzipIO = gzip.GzipFile('rb', fileobj=iobuffer) + contents = gzipIO.read() + iobuffer.close() + logging.info('Contents: %s' % contents) + + callback(ParseContents(contents)) + + +@gen.engine +def DetermineStartDate(client, job, callback): + """If smart_scan is true, lookup the start date from previous job summaries, otherwise use --start_date. + --start_date and job summary days are of the form YYYY-MM-DD. + """ + start_date = options.options.start_date + + # Lookup previous runs started in the last week. + if options.options.smart_scan: + # Search for successful full-scan run in the last week. + last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day') + if last_run is None: + logging.info('No previous successful scan found, rerun with --start_date') + callback(None) + return + + last_run_start = last_run['start_time'] + if (last_run_start + options.options.hours_between_runs * constants.SECONDS_PER_HOUR > time.time()): + logging.info('Last successful run started at %s, less than %d hours ago; skipping.' % + (time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs)) + callback(None) + return + + # Start start_date to the last processed day + 1. + last_day = last_run['stats.last_day'] + start_time = util.ISO8601ToUTCTimestamp(last_day) + constants.SECONDS_PER_DAY + start_date = util.TimestampUTCToISO8601(start_time) + logging.info('Last successful run (%s) scanned up to %s, setting start date to %s' % + (time.asctime(time.localtime(last_run_start)), last_day, start_date)) + + callback(start_date) + + +@gen.engine +def RunOnce(client, job, apple_id, password, callback): + start_date = yield gen.Task(DetermineStartDate, client, job) + if start_date is None: + logging.info('Start date not specified, last run too recent, or smart_scan could not determine a date; exiting.') + callback(None) + return + + query_dates = [] + start_time = util.ISO8601ToUTCTimestamp(start_date) + today = util.NowUTCToISO8601() + while start_time < time.time(): + date = util.TimestampUTCToISO8601(start_time) + if date != today: + query_dates.append(date) + start_time += constants.SECONDS_PER_DAY + + logging.info('fetching data for dates: %r' % query_dates) + results = {} + itc = ITunesTrends(apple_id, password, options.options.vendor_id) + failed = False + for day in query_dates: + if failed: + break + try: + result = yield gen.Task(itc.FetchOneDay, day) + if not result: + # We don't get an exception when iTunesConnect has no data. We also don't want to + # fail as there's no way it will have this data later. + logging.warning('No data for day %s' % day) + else: + results[day] = result + except Exception: + msg = traceback.format_exc() + logging.warning('Error fetching iTunes Connect data for day %s: %s', day, msg) + failed = True + + if len(results) == 0: + callback(None) + else: + # Replace the entire 'itunes' category of previous metrics. This is so we can fix any processing errors we + # may have had. + hms = logs_util.kDailyMetricsTimeByLogType['itunes_trends'] + yield gen.Task(logs_util.UpdateMetrics, client, results, prefix_to_erase='itunes', + dry_run=options.options.dry_run, hms_tuple=hms) + callback(sorted(results.keys())[-1]) + + +@gen.engine +def _Start(callback): + """Grab a lock on job:itunes_trends and call RunOnce. If we get a return value, write it to the job summary.""" + assert options.options.user is not None and options.options.vendor_id is not None + apple_id = '%s@emailscrubbed.com' % options.options.user + # Attempt to lookup iTunes Connect password from secrets. + password = secrets.GetSecret('itunes_connect_%s' % options.options.user) + assert password + + client = db_client.DBClient.Instance() + job = Job(client, 'itunes_trends') + + if options.options.require_lock: + got_lock = yield gen.Task(job.AcquireLock) + if got_lock == False: + logging.warning('Failed to acquire job lock: exiting.') + callback() + return + + result = None + job.Start() + try: + result = yield gen.Task(RunOnce, client, job, apple_id, password) + except: + # Failure: log run summary with trace. + msg = traceback.format_exc() + logging.info('Registering failed run with message: %s' % msg) + yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) + else: + if result is not None and not options.options.dry_run: + # Successful run with data processed and not in dry-run mode: write run summary. + stats = DotDict() + stats['last_day'] = result + logging.info('Registering successful run with stats: %r' % stats) + yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) + finally: + yield gen.Task(job.ReleaseLock) + + callback() + + +if __name__ == '__main__': + sys.exit(main.InitAndRun(_Start)) diff --git a/backend/logs/log_merger.py b/backend/logs/log_merger.py new file mode 100644 index 0000000..bc493bf --- /dev/null +++ b/backend/logs/log_merger.py @@ -0,0 +1,112 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Class to generate a merged log file and upload it to S3. + +Can fetch pre-existing file from S3, append to it locally, and finally upload the result to S3. +Local working file is generated using tempfile.mkstemp and suffixed with the dot-joined id_list. +Name and location in S3 are based on 'id_list' and 's3_base'. +The name of the logfile in S3 will be os.path.join(s3_base, *id_list). +If multiple LocalLogMerge instances are used at the same time, id_list should be unique for each one of them. + +Sample usage: + # S3 file is: /merged_log/viewfinder/full/2013-02-01/i-a5e3f + merge = LocalLogMerge(object_store, ['2013-02-01', 'i-a5e3f'], 'merged_log/viewfinder/full/') + yield gen.Task(merge.FetchExistingFromS3) + + for line in file: + merge.Append(line) # Append a line to the buffer. + if success: + merge.FlushBuffer() # Apply buffer to local file. + else: + merge.DiscardBuffer() # Discard buffer. + # Process another file if needed. + + merge.Close() # Flush buffer and close local file. + yield gen.Task(merge.Upload) # Upload local file to S3 + merge.Cleanup() # Delete local file + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import logging +import os +import tempfile + +from tornado import gen +from viewfinder.backend.base import retry +from viewfinder.backend.storage import file_object_store, s3_object_store + +# Retry policy for uploading files to S3 (merge logs and registry). +kS3UploadRetryPolicy = retry.RetryPolicy(max_tries=5, timeout=300, + min_delay=1, max_delay=30, + check_exception=retry.RetryPolicy.AlwaysRetryOnException) +class LocalLogMerge(object): + """Class used to build a single merged log file locally.""" + + def __init__(self, logs_store, id_list, s3_base): + self._logs_store = logs_store + self._s3_filename = os.path.join(s3_base, *id_list) + fd, self._working_filename = tempfile.mkstemp(suffix='.' + '.'.join(id_list)) + self._output = os.fdopen(fd, 'w') + self._buffer = [] + self._needs_separator = False + + @gen.engine + def FetchExistingFromS3(self, callback): + """If S3 already has a file for this day/instance, fetch it and write its contents to the + local working file. + """ + contents = yield gen.Task(self._logs_store.Get, self._s3_filename, must_exist=False) + if contents is not None: + logging.info('Fetched %d bytes from existing S3 merged log file %s' % (len(contents), self._s3_filename)) + self._output.write(contents) + self._output.flush() + self._needs_separator = True + callback() + + def Append(self, entry): + """Add a single entry to the buffer.""" + assert self._output is not None + self._buffer.append(entry) + + def FlushBuffer(self): + """Write out all entries in the buffer.""" + assert self._output is not None + if not self._buffer: + return + for entry in self._buffer: + if self._needs_separator: + self._output.write('\n') + self._needs_separator = True + self._output.write(entry) + self._output.flush() + self._buffer = [] + + def DiscardBuffer(self): + """Discard all entries in the buffer.""" + self._buffer = [] + + def Close(self): + """Close the working file.""" + assert self._output is not None + self.FlushBuffer() + self._output.close() + self._output = None + + @gen.engine + def Upload(self, callback): + """Upload working file to S3.""" + assert self._output is None, 'Upload called before Close.' + contents = open(self._working_filename, 'r').read() + # Assume 1MB/s transfer speed. If we don't have that good a connection, we really shouldn't be uploading big files. + timeout = max(20.0, len(contents) / 1024 * 1024) + yield gen.Task(retry.CallWithRetryAsync, kS3UploadRetryPolicy, + self._logs_store.Put, self._s3_filename, contents, request_timeout=timeout) + logging.info('Uploaded %d bytes to S3 file %s' % (len(contents), self._s3_filename)) + + callback() + + def Cleanup(self): + """Delete the local working file.""" + os.unlink(self._working_filename) diff --git a/backend/logs/logs_util.py b/backend/logs/logs_util.py new file mode 100644 index 0000000..1618ec9 --- /dev/null +++ b/backend/logs/logs_util.py @@ -0,0 +1,529 @@ +# Copyright 2012 Viewfinder Inc. All Rights Reserved. + +"""Utility functions to handle server logs and metrics. + +ServerLogPaths and UserAnalyticsLogsPaths: classes to handle various paths to logs, and path parsing. + +IsEC2Instance: return true if instance is an AWS instance name + +# Server log contents parsing. +ParseLogLine: parse a raw log line. +ParseSuccessMsg: parse the message logged by user_op_managed SUCCESS. + +# Registry of processed files. +GetRegistry: read a "file registry" from a given path. +WriteRegistry: write a "file registry" to a given path. + +ListClientLogUsers: return the list of users in the client logs repository. + +UpdateMetrics: update the metrics in dynamodb. Merges with existing metrics. + +""" + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import cStringIO +import json +import logging +import os +import re + +from collections import Counter, defaultdict +from tornado import gen + +from viewfinder.backend.base import retry, util +from viewfinder.backend.base.dotdict import DotDict +from viewfinder.backend.db import metric +from viewfinder.backend.storage import file_object_store, s3_object_store, store_utils +from viewfinder.backend.storage.object_store import ObjectStore + +# AWS production instance names. +# TODO(marc): how reliable is this? +kEC2InstanceRe = r'(i-[a-f0-9]+)$' + +kDayRe = r'(\d{4})-(\d{2})-(\d{2})' +kTimeRe = r'(\d{2}):(\d{2}):(\d{2}):(\d{3})' + +################## Server log regexps. ################### +# Some regular expressions to parse log file entries. We don't bother using a compiled version of each +# since python caches the last 100 regexps used on match() or search(). +# Single log line. extracts (date, time, pid, module, message). +kLineRe = r'([-0-9]+) ([-:\.0-9]+) (\[pid:\d+\]) (\w+:\d+): ([^\n]+$)' +# Date from server log file names. Extract (date, time). +kDateRe = r'(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}.\d+)$' +# Message part of a log line for user_op_manager SUCCESS. Extracts (user, device, op, class, method_name). +kSuccessMsgRe = r'SUCCESS: user: (\d+), device: (\d+), op: ([-_0-9a-zA-Z]+), method: (\w+)\.(\w+) .*$' +# Message part of a log line for user_op_manager EXECUTE. Extracts (user, device, op, class, method_name, request). +kExecuteMsgRe = r'EXECUTE: user: (\d+), device: (\d+), op: ([-_0-9a-zA-Z]+), method: (\w+)\.(\w+): (.*)$' +# Message part of a log line for user_op_manager ABORT. Extracts (user, device, op, class, method_name, request). +kAbortMsgRe = r'ABORT: user: (\d+), device: (\d+), op: ([-_0-9a-zA-Z]+), method: (\w+)\.(\w+) (.*)$' +# Ping request with full message. Extract the request dict. +kPingMsgRe = r'/ping OK: request: (.*)$' +# Ping request with full message (using new format). Extract the request and response dicts. +kNewPingMsgRe = r'ping OK: request: (.*) response: (.*)$' +# Code location line of a trace dump. Used to split the stack trace. +kTraceCodeLine = r'( File "/home/[^"]+", line [0-9]+, in [^ ]+)' +# Code location line of a trace dump. Used to extract the file, line and method. +kCodeLocationLine = r' File "([^"]+)", line ([0-9]+), in ([^ ]+)' + +################## Client log regexps. ################### +# Full path to a client-generated log file. //dev--
a"; + + var all = div.getElementsByTagName("*"), + a = div.getElementsByTagName("a")[0]; + + // Can't get basic test support + if ( !all || !all.length || !a ) { + return; + } + + jQuery.support = { + // IE strips leading whitespace when .innerHTML is used + leadingWhitespace: div.firstChild.nodeType === 3, + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + tbody: !div.getElementsByTagName("tbody").length, + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + htmlSerialize: !!div.getElementsByTagName("link").length, + + // Get the style information from getAttribute + // (IE uses .cssText insted) + style: /red/.test( a.getAttribute("style") ), + + // Make sure that URLs aren't manipulated + // (IE normalizes it by default) + hrefNormalized: a.getAttribute("href") === "/a", + + // Make sure that element opacity exists + // (IE uses filter instead) + // Use a regex to work around a WebKit issue. See #5145 + opacity: /^0.55$/.test( a.style.opacity ), + + // Verify style float existence + // (IE uses styleFloat instead of cssFloat) + cssFloat: !!a.style.cssFloat, + + // Make sure that if no value is specified for a checkbox + // that it defaults to "on". + // (WebKit defaults to "" instead) + checkOn: div.getElementsByTagName("input")[0].value === "on", + + // Make sure that a selected-by-default option has a working selected property. + // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) + optSelected: document.createElement("select").appendChild( document.createElement("option") ).selected, + + parentNode: div.removeChild( div.appendChild( document.createElement("div") ) ).parentNode === null, + + // Will be defined later + deleteExpando: true, + checkClone: false, + scriptEval: false, + noCloneEvent: true, + boxModel: null + }; + + script.type = "text/javascript"; + try { + script.appendChild( document.createTextNode( "window." + id + "=1;" ) ); + } catch(e) {} + + root.insertBefore( script, root.firstChild ); + + // Make sure that the execution of code works by injecting a script + // tag with appendChild/createTextNode + // (IE doesn't support this, fails, and uses .text instead) + if ( window[ id ] ) { + jQuery.support.scriptEval = true; + delete window[ id ]; + } + + // Test to see if it's possible to delete an expando from an element + // Fails in Internet Explorer + try { + delete script.test; + + } catch(e) { + jQuery.support.deleteExpando = false; + } + + root.removeChild( script ); + + if ( div.attachEvent && div.fireEvent ) { + div.attachEvent("onclick", function click() { + // Cloning a node shouldn't copy over any + // bound event handlers (IE does this) + jQuery.support.noCloneEvent = false; + div.detachEvent("onclick", click); + }); + div.cloneNode(true).fireEvent("onclick"); + } + + div = document.createElement("div"); + div.innerHTML = ""; + + var fragment = document.createDocumentFragment(); + fragment.appendChild( div.firstChild ); + + // WebKit doesn't clone checked state correctly in fragments + jQuery.support.checkClone = fragment.cloneNode(true).cloneNode(true).lastChild.checked; + + // Figure out if the W3C box model works as expected + // document.body must exist before we can do this + jQuery(function() { + var div = document.createElement("div"); + div.style.width = div.style.paddingLeft = "1px"; + + document.body.appendChild( div ); + jQuery.boxModel = jQuery.support.boxModel = div.offsetWidth === 2; + document.body.removeChild( div ).style.display = 'none'; + + div = null; + }); + + // Technique from Juriy Zaytsev + // http://thinkweb2.com/projects/prototype/detecting-event-support-without-browser-sniffing/ + var eventSupported = function( eventName ) { + var el = document.createElement("div"); + eventName = "on" + eventName; + + var isSupported = (eventName in el); + if ( !isSupported ) { + el.setAttribute(eventName, "return;"); + isSupported = typeof el[eventName] === "function"; + } + el = null; + + return isSupported; + }; + + jQuery.support.submitBubbles = eventSupported("submit"); + jQuery.support.changeBubbles = eventSupported("change"); + + // release memory in IE + root = script = div = all = a = null; +})(); + +jQuery.props = { + "for": "htmlFor", + "class": "className", + readonly: "readOnly", + maxlength: "maxLength", + cellspacing: "cellSpacing", + rowspan: "rowSpan", + colspan: "colSpan", + tabindex: "tabIndex", + usemap: "useMap", + frameborder: "frameBorder" +}; +var expando = "jQuery" + now(), uuid = 0, windowData = {}; + +jQuery.extend({ + cache: {}, + + expando:expando, + + // The following elements throw uncatchable exceptions if you + // attempt to add expando properties to them. + noData: { + "embed": true, + "object": true, + "applet": true + }, + + data: function( elem, name, data ) { + if ( elem.nodeName && jQuery.noData[elem.nodeName.toLowerCase()] ) { + return; + } + + elem = elem == window ? + windowData : + elem; + + var id = elem[ expando ], cache = jQuery.cache, thisCache; + + if ( !id && typeof name === "string" && data === undefined ) { + return null; + } + + // Compute a unique ID for the element + if ( !id ) { + id = ++uuid; + } + + // Avoid generating a new cache unless none exists and we + // want to manipulate it. + if ( typeof name === "object" ) { + elem[ expando ] = id; + thisCache = cache[ id ] = jQuery.extend(true, {}, name); + + } else if ( !cache[ id ] ) { + elem[ expando ] = id; + cache[ id ] = {}; + } + + thisCache = cache[ id ]; + + // Prevent overriding the named cache with undefined values + if ( data !== undefined ) { + thisCache[ name ] = data; + } + + return typeof name === "string" ? thisCache[ name ] : thisCache; + }, + + removeData: function( elem, name ) { + if ( elem.nodeName && jQuery.noData[elem.nodeName.toLowerCase()] ) { + return; + } + + elem = elem == window ? + windowData : + elem; + + var id = elem[ expando ], cache = jQuery.cache, thisCache = cache[ id ]; + + // If we want to remove a specific section of the element's data + if ( name ) { + if ( thisCache ) { + // Remove the section of cache data + delete thisCache[ name ]; + + // If we've removed all the data, remove the element's cache + if ( jQuery.isEmptyObject(thisCache) ) { + jQuery.removeData( elem ); + } + } + + // Otherwise, we want to remove all of the element's data + } else { + if ( jQuery.support.deleteExpando ) { + delete elem[ jQuery.expando ]; + + } else if ( elem.removeAttribute ) { + elem.removeAttribute( jQuery.expando ); + } + + // Completely remove the data cache + delete cache[ id ]; + } + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + if ( typeof key === "undefined" && this.length ) { + return jQuery.data( this[0] ); + + } else if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + var parts = key.split("."); + parts[1] = parts[1] ? "." + parts[1] : ""; + + if ( value === undefined ) { + var data = this.triggerHandler("getData" + parts[1] + "!", [parts[0]]); + + if ( data === undefined && this.length ) { + data = jQuery.data( this[0], key ); + } + return data === undefined && parts[1] ? + this.data( parts[0] ) : + data; + } else { + return this.trigger("setData" + parts[1] + "!", [parts[0], value]).each(function() { + jQuery.data( this, key, value ); + }); + } + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); +jQuery.extend({ + queue: function( elem, type, data ) { + if ( !elem ) { + return; + } + + type = (type || "fx") + "queue"; + var q = jQuery.data( elem, type ); + + // Speed up dequeue by getting out quickly if this is just a lookup + if ( !data ) { + return q || []; + } + + if ( !q || jQuery.isArray(data) ) { + q = jQuery.data( elem, type, jQuery.makeArray(data) ); + + } else { + q.push( data ); + } + + return q; + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), fn = queue.shift(); + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + } + + if ( fn ) { + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift("inprogress"); + } + + fn.call(elem, function() { + jQuery.dequeue(elem, type); + }); + } + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + } + + if ( data === undefined ) { + return jQuery.queue( this[0], type ); + } + return this.each(function( i, elem ) { + var queue = jQuery.queue( this, type, data ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + + // Based off of the plugin by Clint Helfers, with permission. + // http://blindsignals.com/index.php/2009/07/jquery-delay/ + delay: function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[time] || time : time; + type = type || "fx"; + + return this.queue( type, function() { + var elem = this; + setTimeout(function() { + jQuery.dequeue( elem, type ); + }, time ); + }); + }, + + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + } +}); +var rclass = /[\n\t]/g, + rspace = /\s+/, + rreturn = /\r/g, + rspecialurl = /href|src|style/, + rtype = /(button|input)/i, + rfocusable = /(button|input|object|select|textarea)/i, + rclickable = /^(a|area)$/i, + rradiocheck = /radio|checkbox/; + +jQuery.fn.extend({ + attr: function( name, value ) { + return access( this, name, value, true, jQuery.attr ); + }, + + removeAttr: function( name, fn ) { + return this.each(function(){ + jQuery.attr( this, name, "" ); + if ( this.nodeType === 1 ) { + this.removeAttribute( name ); + } + }); + }, + + addClass: function( value ) { + if ( jQuery.isFunction(value) ) { + return this.each(function(i) { + var self = jQuery(this); + self.addClass( value.call(this, i, self.attr("class")) ); + }); + } + + if ( value && typeof value === "string" ) { + var classNames = (value || "").split( rspace ); + + for ( var i = 0, l = this.length; i < l; i++ ) { + var elem = this[i]; + + if ( elem.nodeType === 1 ) { + if ( !elem.className ) { + elem.className = value; + + } else { + var className = " " + elem.className + " ", setClass = elem.className; + for ( var c = 0, cl = classNames.length; c < cl; c++ ) { + if ( className.indexOf( " " + classNames[c] + " " ) < 0 ) { + setClass += " " + classNames[c]; + } + } + elem.className = jQuery.trim( setClass ); + } + } + } + } + + return this; + }, + + removeClass: function( value ) { + if ( jQuery.isFunction(value) ) { + return this.each(function(i) { + var self = jQuery(this); + self.removeClass( value.call(this, i, self.attr("class")) ); + }); + } + + if ( (value && typeof value === "string") || value === undefined ) { + var classNames = (value || "").split(rspace); + + for ( var i = 0, l = this.length; i < l; i++ ) { + var elem = this[i]; + + if ( elem.nodeType === 1 && elem.className ) { + if ( value ) { + var className = (" " + elem.className + " ").replace(rclass, " "); + for ( var c = 0, cl = classNames.length; c < cl; c++ ) { + className = className.replace(" " + classNames[c] + " ", " "); + } + elem.className = jQuery.trim( className ); + + } else { + elem.className = ""; + } + } + } + } + + return this; + }, + + toggleClass: function( value, stateVal ) { + var type = typeof value, isBool = typeof stateVal === "boolean"; + + if ( jQuery.isFunction( value ) ) { + return this.each(function(i) { + var self = jQuery(this); + self.toggleClass( value.call(this, i, self.attr("class"), stateVal), stateVal ); + }); + } + + return this.each(function() { + if ( type === "string" ) { + // toggle individual class names + var className, i = 0, self = jQuery(this), + state = stateVal, + classNames = value.split( rspace ); + + while ( (className = classNames[ i++ ]) ) { + // check each className given, space seperated list + state = isBool ? state : !self.hasClass( className ); + self[ state ? "addClass" : "removeClass" ]( className ); + } + + } else if ( type === "undefined" || type === "boolean" ) { + if ( this.className ) { + // store className if set + jQuery.data( this, "__className__", this.className ); + } + + // toggle whole className + this.className = this.className || value === false ? "" : jQuery.data( this, "__className__" ) || ""; + } + }); + }, + + hasClass: function( selector ) { + var className = " " + selector + " "; + for ( var i = 0, l = this.length; i < l; i++ ) { + if ( (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) { + return true; + } + } + + return false; + }, + + val: function( value ) { + if ( value === undefined ) { + var elem = this[0]; + + if ( elem ) { + if ( jQuery.nodeName( elem, "option" ) ) { + return (elem.attributes.value || {}).specified ? elem.value : elem.text; + } + + // We need to handle select boxes special + if ( jQuery.nodeName( elem, "select" ) ) { + var index = elem.selectedIndex, + values = [], + options = elem.options, + one = elem.type === "select-one"; + + // Nothing was selected + if ( index < 0 ) { + return null; + } + + // Loop through all the selected options + for ( var i = one ? index : 0, max = one ? index + 1 : options.length; i < max; i++ ) { + var option = options[ i ]; + + if ( option.selected ) { + // Get the specifc value for the option + value = jQuery(option).val(); + + // We don't need an array for one selects + if ( one ) { + return value; + } + + // Multi-Selects return an array + values.push( value ); + } + } + + return values; + } + + // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified + if ( rradiocheck.test( elem.type ) && !jQuery.support.checkOn ) { + return elem.getAttribute("value") === null ? "on" : elem.value; + } + + + // Everything else, we just grab the value + return (elem.value || "").replace(rreturn, ""); + + } + + return undefined; + } + + var isFunction = jQuery.isFunction(value); + + return this.each(function(i) { + var self = jQuery(this), val = value; + + if ( this.nodeType !== 1 ) { + return; + } + + if ( isFunction ) { + val = value.call(this, i, self.val()); + } + + // Typecast each time if the value is a Function and the appended + // value is therefore different each time. + if ( typeof val === "number" ) { + val += ""; + } + + if ( jQuery.isArray(val) && rradiocheck.test( this.type ) ) { + this.checked = jQuery.inArray( self.val(), val ) >= 0; + + } else if ( jQuery.nodeName( this, "select" ) ) { + var values = jQuery.makeArray(val); + + jQuery( "option", this ).each(function() { + this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0; + }); + + if ( !values.length ) { + this.selectedIndex = -1; + } + + } else { + this.value = val; + } + }); + } +}); + +jQuery.extend({ + attrFn: { + val: true, + css: true, + html: true, + text: true, + data: true, + width: true, + height: true, + offset: true + }, + + attr: function( elem, name, value, pass ) { + // don't set attributes on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 ) { + return undefined; + } + + if ( pass && name in jQuery.attrFn ) { + return jQuery(elem)[name](value); + } + + var notxml = elem.nodeType !== 1 || !jQuery.isXMLDoc( elem ), + // Whether we are setting (or getting) + set = value !== undefined; + + // Try to normalize/fix the name + name = notxml && jQuery.props[ name ] || name; + + // Only do all the following if this is a node (faster for style) + if ( elem.nodeType === 1 ) { + // These attributes require special treatment + var special = rspecialurl.test( name ); + + // Safari mis-reports the default selected property of an option + // Accessing the parent's selectedIndex property fixes it + if ( name === "selected" && !jQuery.support.optSelected ) { + var parent = elem.parentNode; + if ( parent ) { + parent.selectedIndex; + + // Make sure that it also works with optgroups, see #5701 + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + } + + // If applicable, access the attribute via the DOM 0 way + if ( name in elem && notxml && !special ) { + if ( set ) { + // We can't allow the type property to be changed (since it causes problems in IE) + if ( name === "type" && rtype.test( elem.nodeName ) && elem.parentNode ) { + jQuery.error( "type property can't be changed" ); + } + + elem[ name ] = value; + } + + // browsers index elements by id/name on forms, give priority to attributes. + if ( jQuery.nodeName( elem, "form" ) && elem.getAttributeNode(name) ) { + return elem.getAttributeNode( name ).nodeValue; + } + + // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set + // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + if ( name === "tabIndex" ) { + var attributeNode = elem.getAttributeNode( "tabIndex" ); + + return attributeNode && attributeNode.specified ? + attributeNode.value : + rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? + 0 : + undefined; + } + + return elem[ name ]; + } + + if ( !jQuery.support.style && notxml && name === "style" ) { + if ( set ) { + elem.style.cssText = "" + value; + } + + return elem.style.cssText; + } + + if ( set ) { + // convert the value to a string (all browsers do this but IE) see #1070 + elem.setAttribute( name, "" + value ); + } + + var attr = !jQuery.support.hrefNormalized && notxml && special ? + // Some attributes require a special call on IE + elem.getAttribute( name, 2 ) : + elem.getAttribute( name ); + + // Non-existent attributes return null, we normalize to undefined + return attr === null ? undefined : attr; + } + + // elem is actually elem.style ... set the style + // Using attr for specific style information is now deprecated. Use style instead. + return jQuery.style( elem, name, value ); + } +}); +var rnamespaces = /\.(.*)$/, + fcleanup = function( nm ) { + return nm.replace(/[^\w\s\.\|`]/g, function( ch ) { + return "\\" + ch; + }); + }; + +/* + * A number of helper functions used for managing events. + * Many of the ideas behind this code originated from + * Dean Edwards' addEvent library. + */ +jQuery.event = { + + // Bind an event to an element + // Original by Dean Edwards + add: function( elem, types, handler, data ) { + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + // For whatever reason, IE has trouble passing the window object + // around, causing it to be cloned in the process + if ( elem.setInterval && ( elem !== window && !elem.frameElement ) ) { + elem = window; + } + + var handleObjIn, handleObj; + + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + } + + // Make sure that the function being executed has a unique ID + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure + var elemData = jQuery.data( elem ); + + // If no elemData is found then we must be trying to bind to one of the + // banned noData elements + if ( !elemData ) { + return; + } + + var events = elemData.events = elemData.events || {}, + eventHandle = elemData.handle, eventHandle; + + if ( !eventHandle ) { + elemData.handle = eventHandle = function() { + // Handle the second event of a trigger and when + // an event is called after a page has unloaded + return typeof jQuery !== "undefined" && !jQuery.event.triggered ? + jQuery.event.handle.apply( eventHandle.elem, arguments ) : + undefined; + }; + } + + // Add elem as a property of the handle function + // This is to prevent a memory leak with non-native events in IE. + eventHandle.elem = elem; + + // Handle multiple events separated by a space + // jQuery(...).bind("mouseover mouseout", fn); + types = types.split(" "); + + var type, i = 0, namespaces; + + while ( (type = types[ i++ ]) ) { + handleObj = handleObjIn ? + jQuery.extend({}, handleObjIn) : + { handler: handler, data: data }; + + // Namespaced event handlers + if ( type.indexOf(".") > -1 ) { + namespaces = type.split("."); + type = namespaces.shift(); + handleObj.namespace = namespaces.slice(0).sort().join("."); + + } else { + namespaces = []; + handleObj.namespace = ""; + } + + handleObj.type = type; + handleObj.guid = handler.guid; + + // Get the current list of functions bound to this event + var handlers = events[ type ], + special = jQuery.event.special[ type ] || {}; + + // Init the event handler queue + if ( !handlers ) { + handlers = events[ type ] = []; + + // Check for a special event handler + // Only use addEventListener/attachEvent if the special + // events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add the function to the element's handler list + handlers.push( handleObj ); + + // Keep track of which events have been used, for global triggering + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + global: {}, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, pos ) { + // don't do events on text and comment nodes + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + var ret, type, fn, i = 0, all, namespaces, namespace, special, eventType, handleObj, origType, + elemData = jQuery.data( elem ), + events = elemData && elemData.events; + + if ( !elemData || !events ) { + return; + } + + // types is actually an event object here + if ( types && types.type ) { + handler = types.handler; + types = types.type; + } + + // Unbind all events for the element + if ( !types || typeof types === "string" && types.charAt(0) === "." ) { + types = types || ""; + + for ( type in events ) { + jQuery.event.remove( elem, type + types ); + } + + return; + } + + // Handle multiple events separated by a space + // jQuery(...).unbind("mouseover mouseout", fn); + types = types.split(" "); + + while ( (type = types[ i++ ]) ) { + origType = type; + handleObj = null; + all = type.indexOf(".") < 0; + namespaces = []; + + if ( !all ) { + // Namespaced event handlers + namespaces = type.split("."); + type = namespaces.shift(); + + namespace = new RegExp("(^|\\.)" + + jQuery.map( namespaces.slice(0).sort(), fcleanup ).join("\\.(?:.*\\.)?") + "(\\.|$)") + } + + eventType = events[ type ]; + + if ( !eventType ) { + continue; + } + + if ( !handler ) { + for ( var j = 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( all || namespace.test( handleObj.namespace ) ) { + jQuery.event.remove( elem, origType, handleObj.handler, j ); + eventType.splice( j--, 1 ); + } + } + + continue; + } + + special = jQuery.event.special[ type ] || {}; + + for ( var j = pos || 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( handler.guid === handleObj.guid ) { + // remove the given handler for the given type + if ( all || namespace.test( handleObj.namespace ) ) { + if ( pos == null ) { + eventType.splice( j--, 1 ); + } + + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + + if ( pos != null ) { + break; + } + } + } + + // remove generic event handler if no more handlers exist + if ( eventType.length === 0 || pos != null && eventType.length === 1 ) { + if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) { + removeEvent( elem, type, elemData.handle ); + } + + ret = null; + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + var handle = elemData.handle; + if ( handle ) { + handle.elem = null; + } + + delete elemData.events; + delete elemData.handle; + + if ( jQuery.isEmptyObject( elemData ) ) { + jQuery.removeData( elem ); + } + } + }, + + // bubbling is internal + trigger: function( event, data, elem /*, bubbling */ ) { + // Event object or event type + var type = event.type || event, + bubbling = arguments[3]; + + if ( !bubbling ) { + event = typeof event === "object" ? + // jQuery.Event object + event[expando] ? event : + // Object literal + jQuery.extend( jQuery.Event(type), event ) : + // Just the event type (string) + jQuery.Event(type); + + if ( type.indexOf("!") >= 0 ) { + event.type = type = type.slice(0, -1); + event.exclusive = true; + } + + // Handle a global trigger + if ( !elem ) { + // Don't bubble custom events when global (to avoid too much overhead) + event.stopPropagation(); + + // Only trigger if we've ever bound an event for it + if ( jQuery.event.global[ type ] ) { + jQuery.each( jQuery.cache, function() { + if ( this.events && this.events[type] ) { + jQuery.event.trigger( event, data, this.handle.elem ); + } + }); + } + } + + // Handle triggering a single element + + // don't do events on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 ) { + return undefined; + } + + // Clean up in case it is reused + event.result = undefined; + event.target = elem; + + // Clone the incoming data, if any + data = jQuery.makeArray( data ); + data.unshift( event ); + } + + event.currentTarget = elem; + + // Trigger the event, it is assumed that "handle" is a function + var handle = jQuery.data( elem, "handle" ); + if ( handle ) { + handle.apply( elem, data ); + } + + var parent = elem.parentNode || elem.ownerDocument; + + // Trigger an inline bound script + try { + if ( !(elem && elem.nodeName && jQuery.noData[elem.nodeName.toLowerCase()]) ) { + if ( elem[ "on" + type ] && elem[ "on" + type ].apply( elem, data ) === false ) { + event.result = false; + } + } + + // prevent IE from throwing an error for some elements with some event types, see #3533 + } catch (e) {} + + if ( !event.isPropagationStopped() && parent ) { + jQuery.event.trigger( event, data, parent, true ); + + } else if ( !event.isDefaultPrevented() ) { + var target = event.target, old, + isClick = jQuery.nodeName(target, "a") && type === "click", + special = jQuery.event.special[ type ] || {}; + + if ( (!special._default || special._default.call( elem, event ) === false) && + !isClick && !(target && target.nodeName && jQuery.noData[target.nodeName.toLowerCase()]) ) { + + try { + if ( target[ type ] ) { + // Make sure that we don't accidentally re-trigger the onFOO events + old = target[ "on" + type ]; + + if ( old ) { + target[ "on" + type ] = null; + } + + jQuery.event.triggered = true; + target[ type ](); + } + + // prevent IE from throwing an error for some elements with some event types, see #3533 + } catch (e) {} + + if ( old ) { + target[ "on" + type ] = old; + } + + jQuery.event.triggered = false; + } + } + }, + + handle: function( event ) { + var all, handlers, namespaces, namespace, events; + + event = arguments[0] = jQuery.event.fix( event || window.event ); + event.currentTarget = this; + + // Namespaced event handlers + all = event.type.indexOf(".") < 0 && !event.exclusive; + + if ( !all ) { + namespaces = event.type.split("."); + event.type = namespaces.shift(); + namespace = new RegExp("(^|\\.)" + namespaces.slice(0).sort().join("\\.(?:.*\\.)?") + "(\\.|$)"); + } + + var events = jQuery.data(this, "events"), handlers = events[ event.type ]; + + if ( events && handlers ) { + // Clone the handlers to prevent manipulation + handlers = handlers.slice(0); + + for ( var j = 0, l = handlers.length; j < l; j++ ) { + var handleObj = handlers[ j ]; + + // Filter the functions by class + if ( all || namespace.test( handleObj.namespace ) ) { + // Pass in a reference to the handler function itself + // So that we can later remove it + event.handler = handleObj.handler; + event.data = handleObj.data; + event.handleObj = handleObj; + + var ret = handleObj.handler.apply( this, arguments ); + + if ( ret !== undefined ) { + event.result = ret; + if ( ret === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + + if ( event.isImmediatePropagationStopped() ) { + break; + } + } + } + } + + return event.result; + }, + + props: "altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "), + + fix: function( event ) { + if ( event[ expando ] ) { + return event; + } + + // store a copy of the original event object + // and "clone" to set read-only properties + var originalEvent = event; + event = jQuery.Event( originalEvent ); + + for ( var i = this.props.length, prop; i; ) { + prop = this.props[ --i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Fix target property, if necessary + if ( !event.target ) { + event.target = event.srcElement || document; // Fixes #1925 where srcElement might not be defined either + } + + // check if target is a textnode (safari) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && event.fromElement ) { + event.relatedTarget = event.fromElement === event.target ? event.toElement : event.fromElement; + } + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && event.clientX != null ) { + var doc = document.documentElement, body = document.body; + event.pageX = event.clientX + (doc && doc.scrollLeft || body && body.scrollLeft || 0) - (doc && doc.clientLeft || body && body.clientLeft || 0); + event.pageY = event.clientY + (doc && doc.scrollTop || body && body.scrollTop || 0) - (doc && doc.clientTop || body && body.clientTop || 0); + } + + // Add which for key events + if ( !event.which && ((event.charCode || event.charCode === 0) ? event.charCode : event.keyCode) ) { + event.which = event.charCode || event.keyCode; + } + + // Add metaKey to non-Mac browsers (use ctrl for PC's and Meta for Macs) + if ( !event.metaKey && event.ctrlKey ) { + event.metaKey = event.ctrlKey; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && event.button !== undefined ) { + event.which = (event.button & 1 ? 1 : ( event.button & 2 ? 3 : ( event.button & 4 ? 2 : 0 ) )); + } + + return event; + }, + + // Deprecated, use jQuery.guid instead + guid: 1E8, + + // Deprecated, use jQuery.proxy instead + proxy: jQuery.proxy, + + special: { + ready: { + // Make sure the ready event is setup + setup: jQuery.bindReady, + teardown: jQuery.noop + }, + + live: { + add: function( handleObj ) { + jQuery.event.add( this, handleObj.origType, jQuery.extend({}, handleObj, {handler: liveHandler}) ); + }, + + remove: function( handleObj ) { + var remove = true, + type = handleObj.origType.replace(rnamespaces, ""); + + jQuery.each( jQuery.data(this, "events").live || [], function() { + if ( type === this.origType.replace(rnamespaces, "") ) { + remove = false; + return false; + } + }); + + if ( remove ) { + jQuery.event.remove( this, handleObj.origType, liveHandler ); + } + } + + }, + + beforeunload: { + setup: function( data, namespaces, eventHandle ) { + // We only want to do this special case on windows + if ( this.setInterval ) { + this.onbeforeunload = eventHandle; + } + + return false; + }, + teardown: function( namespaces, eventHandle ) { + if ( this.onbeforeunload === eventHandle ) { + this.onbeforeunload = null; + } + } + } + } +}; + +var removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + elem.removeEventListener( type, handle, false ); + } : + function( elem, type, handle ) { + elem.detachEvent( "on" + type, handle ); + }; + +jQuery.Event = function( src ) { + // Allow instantiation without the 'new' keyword + if ( !this.preventDefault ) { + return new jQuery.Event( src ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + // Event type + } else { + this.type = src; + } + + // timeStamp is buggy for some events on Firefox(#3843) + // So we won't rely on the native value + this.timeStamp = now(); + + // Mark it as fixed + this[ expando ] = true; +}; + +function returnFalse() { + return false; +} +function returnTrue() { + return true; +} + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + preventDefault: function() { + this.isDefaultPrevented = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + + // if preventDefault exists run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + } + // otherwise set the returnValue property of the original event to false (IE) + e.returnValue = false; + }, + stopPropagation: function() { + this.isPropagationStopped = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + // if stopPropagation exists run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + // otherwise set the cancelBubble property of the original event to true (IE) + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + }, + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse +}; + +// Checks if an event happened on an element within another element +// Used in jQuery.event.special.mouseenter and mouseleave handlers +var withinElement = function( event ) { + // Check if mouse(over|out) are still within the same parent element + var parent = event.relatedTarget; + + // Firefox sometimes assigns relatedTarget a XUL element + // which we cannot access the parentNode property of + try { + // Traverse up the tree + while ( parent && parent !== this ) { + parent = parent.parentNode; + } + + if ( parent !== this ) { + // set the correct event type + event.type = event.data; + + // handle event if we actually just moused on to a non sub-element + jQuery.event.handle.apply( this, arguments ); + } + + // assuming we've left the element since we most likely mousedover a xul element + } catch(e) { } +}, + +// In case of event delegation, we only need to rename the event.type, +// liveHandler will take care of the rest. +delegate = function( event ) { + event.type = event.data; + jQuery.event.handle.apply( this, arguments ); +}; + +// Create mouseenter and mouseleave events +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + setup: function( data ) { + jQuery.event.add( this, fix, data && data.selector ? delegate : withinElement, orig ); + }, + teardown: function( data ) { + jQuery.event.remove( this, fix, data && data.selector ? delegate : withinElement ); + } + }; +}); + +// submit delegation +if ( !jQuery.support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function( data, namespaces ) { + if ( this.nodeName.toLowerCase() !== "form" ) { + jQuery.event.add(this, "click.specialSubmit", function( e ) { + var elem = e.target, type = elem.type; + + if ( (type === "submit" || type === "image") && jQuery( elem ).closest("form").length ) { + return trigger( "submit", this, arguments ); + } + }); + + jQuery.event.add(this, "keypress.specialSubmit", function( e ) { + var elem = e.target, type = elem.type; + + if ( (type === "text" || type === "password") && jQuery( elem ).closest("form").length && e.keyCode === 13 ) { + return trigger( "submit", this, arguments ); + } + }); + + } else { + return false; + } + }, + + teardown: function( namespaces ) { + jQuery.event.remove( this, ".specialSubmit" ); + } + }; + +} + +// change delegation, happens here so we have bind. +if ( !jQuery.support.changeBubbles ) { + + var formElems = /textarea|input|select/i, + + changeFilters, + + getVal = function( elem ) { + var type = elem.type, val = elem.value; + + if ( type === "radio" || type === "checkbox" ) { + val = elem.checked; + + } else if ( type === "select-multiple" ) { + val = elem.selectedIndex > -1 ? + jQuery.map( elem.options, function( elem ) { + return elem.selected; + }).join("-") : + ""; + + } else if ( elem.nodeName.toLowerCase() === "select" ) { + val = elem.selectedIndex; + } + + return val; + }, + + testChange = function testChange( e ) { + var elem = e.target, data, val; + + if ( !formElems.test( elem.nodeName ) || elem.readOnly ) { + return; + } + + data = jQuery.data( elem, "_change_data" ); + val = getVal(elem); + + // the current data will be also retrieved by beforeactivate + if ( e.type !== "focusout" || elem.type !== "radio" ) { + jQuery.data( elem, "_change_data", val ); + } + + if ( data === undefined || val === data ) { + return; + } + + if ( data != null || val ) { + e.type = "change"; + return jQuery.event.trigger( e, arguments[1], elem ); + } + }; + + jQuery.event.special.change = { + filters: { + focusout: testChange, + + click: function( e ) { + var elem = e.target, type = elem.type; + + if ( type === "radio" || type === "checkbox" || elem.nodeName.toLowerCase() === "select" ) { + return testChange.call( this, e ); + } + }, + + // Change has to be called before submit + // Keydown will be called before keypress, which is used in submit-event delegation + keydown: function( e ) { + var elem = e.target, type = elem.type; + + if ( (e.keyCode === 13 && elem.nodeName.toLowerCase() !== "textarea") || + (e.keyCode === 32 && (type === "checkbox" || type === "radio")) || + type === "select-multiple" ) { + return testChange.call( this, e ); + } + }, + + // Beforeactivate happens also before the previous element is blurred + // with this event you can't trigger a change event, but you can store + // information/focus[in] is not needed anymore + beforeactivate: function( e ) { + var elem = e.target; + jQuery.data( elem, "_change_data", getVal(elem) ); + } + }, + + setup: function( data, namespaces ) { + if ( this.type === "file" ) { + return false; + } + + for ( var type in changeFilters ) { + jQuery.event.add( this, type + ".specialChange", changeFilters[type] ); + } + + return formElems.test( this.nodeName ); + }, + + teardown: function( namespaces ) { + jQuery.event.remove( this, ".specialChange" ); + + return formElems.test( this.nodeName ); + } + }; + + changeFilters = jQuery.event.special.change.filters; +} + +function trigger( type, elem, args ) { + args[0].type = type; + return jQuery.event.handle.apply( elem, args ); +} + +// Create "bubbling" focus and blur events +if ( document.addEventListener ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + jQuery.event.special[ fix ] = { + setup: function() { + this.addEventListener( orig, handler, true ); + }, + teardown: function() { + this.removeEventListener( orig, handler, true ); + } + }; + + function handler( e ) { + e = jQuery.event.fix( e ); + e.type = fix; + return jQuery.event.handle.call( this, e ); + } + }); +} + +jQuery.each(["bind", "one"], function( i, name ) { + jQuery.fn[ name ] = function( type, data, fn ) { + // Handle object literals + if ( typeof type === "object" ) { + for ( var key in type ) { + this[ name ](key, data, type[key], fn); + } + return this; + } + + if ( jQuery.isFunction( data ) ) { + fn = data; + data = undefined; + } + + var handler = name === "one" ? jQuery.proxy( fn, function( event ) { + jQuery( this ).unbind( event, handler ); + return fn.apply( this, arguments ); + }) : fn; + + if ( type === "unload" && name !== "one" ) { + this.one( type, data, fn ); + + } else { + for ( var i = 0, l = this.length; i < l; i++ ) { + jQuery.event.add( this[i], type, handler, data ); + } + } + + return this; + }; +}); + +jQuery.fn.extend({ + unbind: function( type, fn ) { + // Handle object literals + if ( typeof type === "object" && !type.preventDefault ) { + for ( var key in type ) { + this.unbind(key, type[key]); + } + + } else { + for ( var i = 0, l = this.length; i < l; i++ ) { + jQuery.event.remove( this[i], type, fn ); + } + } + + return this; + }, + + delegate: function( selector, types, data, fn ) { + return this.live( types, data, fn, selector ); + }, + + undelegate: function( selector, types, fn ) { + if ( arguments.length === 0 ) { + return this.unbind( "live" ); + + } else { + return this.die( types, null, fn, selector ); + } + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + + triggerHandler: function( type, data ) { + if ( this[0] ) { + var event = jQuery.Event( type ); + event.preventDefault(); + event.stopPropagation(); + jQuery.event.trigger( event, data, this[0] ); + return event.result; + } + }, + + toggle: function( fn ) { + // Save reference to arguments for access in closure + var args = arguments, i = 1; + + // link all the functions, so any of them can unbind this click handler + while ( i < args.length ) { + jQuery.proxy( fn, args[ i++ ] ); + } + + return this.click( jQuery.proxy( fn, function( event ) { + // Figure out which function to execute + var lastToggle = ( jQuery.data( this, "lastToggle" + fn.guid ) || 0 ) % i; + jQuery.data( this, "lastToggle" + fn.guid, lastToggle + 1 ); + + // Make sure that clicks stop + event.preventDefault(); + + // and execute the function + return args[ lastToggle ].apply( this, arguments ) || false; + })); + }, + + hover: function( fnOver, fnOut ) { + return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); + } +}); + +var liveMap = { + focus: "focusin", + blur: "focusout", + mouseenter: "mouseover", + mouseleave: "mouseout" +}; + +jQuery.each(["live", "die"], function( i, name ) { + jQuery.fn[ name ] = function( types, data, fn, origSelector /* Internal Use Only */ ) { + var type, i = 0, match, namespaces, preType, + selector = origSelector || this.selector, + context = origSelector ? this : jQuery( this.context ); + + if ( jQuery.isFunction( data ) ) { + fn = data; + data = undefined; + } + + types = (types || "").split(" "); + + while ( (type = types[ i++ ]) != null ) { + match = rnamespaces.exec( type ); + namespaces = ""; + + if ( match ) { + namespaces = match[0]; + type = type.replace( rnamespaces, "" ); + } + + if ( type === "hover" ) { + types.push( "mouseenter" + namespaces, "mouseleave" + namespaces ); + continue; + } + + preType = type; + + if ( type === "focus" || type === "blur" ) { + types.push( liveMap[ type ] + namespaces ); + type = type + namespaces; + + } else { + type = (liveMap[ type ] || type) + namespaces; + } + + if ( name === "live" ) { + // bind live handler + context.each(function(){ + jQuery.event.add( this, liveConvert( type, selector ), + { data: data, selector: selector, handler: fn, origType: type, origHandler: fn, preType: preType } ); + }); + + } else { + // unbind live handler + context.unbind( liveConvert( type, selector ), fn ); + } + } + + return this; + } +}); + +function liveHandler( event ) { + var stop, elems = [], selectors = [], args = arguments, + related, match, handleObj, elem, j, i, l, data, + events = jQuery.data( this, "events" ); + + // Make sure we avoid non-left-click bubbling in Firefox (#3861) + if ( event.liveFired === this || !events || !events.live || event.button && event.type === "click" ) { + return; + } + + event.liveFired = this; + + var live = events.live.slice(0); + + for ( j = 0; j < live.length; j++ ) { + handleObj = live[j]; + + if ( handleObj.origType.replace( rnamespaces, "" ) === event.type ) { + selectors.push( handleObj.selector ); + + } else { + live.splice( j--, 1 ); + } + } + + match = jQuery( event.target ).closest( selectors, event.currentTarget ); + + for ( i = 0, l = match.length; i < l; i++ ) { + for ( j = 0; j < live.length; j++ ) { + handleObj = live[j]; + + if ( match[i].selector === handleObj.selector ) { + elem = match[i].elem; + related = null; + + // Those two events require additional checking + if ( handleObj.preType === "mouseenter" || handleObj.preType === "mouseleave" ) { + related = jQuery( event.relatedTarget ).closest( handleObj.selector )[0]; + } + + if ( !related || related !== elem ) { + elems.push({ elem: elem, handleObj: handleObj }); + } + } + } + } + + for ( i = 0, l = elems.length; i < l; i++ ) { + match = elems[i]; + event.currentTarget = match.elem; + event.data = match.handleObj.data; + event.handleObj = match.handleObj; + + if ( match.handleObj.origHandler.apply( match.elem, args ) === false ) { + stop = false; + break; + } + } + + return stop; +} + +function liveConvert( type, selector ) { + return "live." + (type && type !== "*" ? type + "." : "") + selector.replace(/\./g, "`").replace(/ /g, "&"); +} + +jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " + + "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + + "change select submit keydown keypress keyup error").split(" "), function( i, name ) { + + // Handle event binding + jQuery.fn[ name ] = function( fn ) { + return fn ? this.bind( name, fn ) : this.trigger( name ); + }; + + if ( jQuery.attrFn ) { + jQuery.attrFn[ name ] = true; + } +}); + +// Prevent memory leaks in IE +// Window isn't included so as not to unbind existing unload events +// More info: +// - http://isaacschlueter.com/2006/10/msie-memory-leaks/ +if ( window.attachEvent && !window.addEventListener ) { + window.attachEvent("onunload", function() { + for ( var id in jQuery.cache ) { + if ( jQuery.cache[ id ].handle ) { + // Try/Catch is to handle iframes being unloaded, see #4280 + try { + jQuery.event.remove( jQuery.cache[ id ].handle.elem ); + } catch(e) {} + } + } + }); +} +/*! + * Sizzle CSS Selector Engine - v1.0 + * Copyright 2009, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * More information: http://sizzlejs.com/ + */ +(function(){ + +var chunker = /((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, + done = 0, + toString = Object.prototype.toString, + hasDuplicate = false, + baseHasDuplicate = true; + +// Here we check if the JavaScript engine is using some sort of +// optimization where it does not always call our comparision +// function. If that is the case, discard the hasDuplicate value. +// Thus far that includes Google Chrome. +[0, 0].sort(function(){ + baseHasDuplicate = false; + return 0; +}); + +var Sizzle = function(selector, context, results, seed) { + results = results || []; + var origContext = context = context || document; + + if ( context.nodeType !== 1 && context.nodeType !== 9 ) { + return []; + } + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + var parts = [], m, set, checkSet, extra, prune = true, contextXML = isXML(context), + soFar = selector; + + // Reset the position of the chunker regexp (start from head) + while ( (chunker.exec(""), m = chunker.exec(soFar)) !== null ) { + soFar = m[3]; + + parts.push( m[1] ); + + if ( m[2] ) { + extra = m[3]; + break; + } + } + + if ( parts.length > 1 && origPOS.exec( selector ) ) { + if ( parts.length === 2 && Expr.relative[ parts[0] ] ) { + set = posProcess( parts[0] + parts[1], context ); + } else { + set = Expr.relative[ parts[0] ] ? + [ context ] : + Sizzle( parts.shift(), context ); + + while ( parts.length ) { + selector = parts.shift(); + + if ( Expr.relative[ selector ] ) { + selector += parts.shift(); + } + + set = posProcess( selector, set ); + } + } + } else { + // Take a shortcut and set the context if the root selector is an ID + // (but not if it'll be faster if the inner selector is an ID) + if ( !seed && parts.length > 1 && context.nodeType === 9 && !contextXML && + Expr.match.ID.test(parts[0]) && !Expr.match.ID.test(parts[parts.length - 1]) ) { + var ret = Sizzle.find( parts.shift(), context, contextXML ); + context = ret.expr ? Sizzle.filter( ret.expr, ret.set )[0] : ret.set[0]; + } + + if ( context ) { + var ret = seed ? + { expr: parts.pop(), set: makeArray(seed) } : + Sizzle.find( parts.pop(), parts.length === 1 && (parts[0] === "~" || parts[0] === "+") && context.parentNode ? context.parentNode : context, contextXML ); + set = ret.expr ? Sizzle.filter( ret.expr, ret.set ) : ret.set; + + if ( parts.length > 0 ) { + checkSet = makeArray(set); + } else { + prune = false; + } + + while ( parts.length ) { + var cur = parts.pop(), pop = cur; + + if ( !Expr.relative[ cur ] ) { + cur = ""; + } else { + pop = parts.pop(); + } + + if ( pop == null ) { + pop = context; + } + + Expr.relative[ cur ]( checkSet, pop, contextXML ); + } + } else { + checkSet = parts = []; + } + } + + if ( !checkSet ) { + checkSet = set; + } + + if ( !checkSet ) { + Sizzle.error( cur || selector ); + } + + if ( toString.call(checkSet) === "[object Array]" ) { + if ( !prune ) { + results.push.apply( results, checkSet ); + } else if ( context && context.nodeType === 1 ) { + for ( var i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && (checkSet[i] === true || checkSet[i].nodeType === 1 && contains(context, checkSet[i])) ) { + results.push( set[i] ); + } + } + } else { + for ( var i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && checkSet[i].nodeType === 1 ) { + results.push( set[i] ); + } + } + } + } else { + makeArray( checkSet, results ); + } + + if ( extra ) { + Sizzle( extra, origContext, results, seed ); + Sizzle.uniqueSort( results ); + } + + return results; +}; + +Sizzle.uniqueSort = function(results){ + if ( sortOrder ) { + hasDuplicate = baseHasDuplicate; + results.sort(sortOrder); + + if ( hasDuplicate ) { + for ( var i = 1; i < results.length; i++ ) { + if ( results[i] === results[i-1] ) { + results.splice(i--, 1); + } + } + } + } + + return results; +}; + +Sizzle.matches = function(expr, set){ + return Sizzle(expr, null, null, set); +}; + +Sizzle.find = function(expr, context, isXML){ + var set, match; + + if ( !expr ) { + return []; + } + + for ( var i = 0, l = Expr.order.length; i < l; i++ ) { + var type = Expr.order[i], match; + + if ( (match = Expr.leftMatch[ type ].exec( expr )) ) { + var left = match[1]; + match.splice(1,1); + + if ( left.substr( left.length - 1 ) !== "\\" ) { + match[1] = (match[1] || "").replace(/\\/g, ""); + set = Expr.find[ type ]( match, context, isXML ); + if ( set != null ) { + expr = expr.replace( Expr.match[ type ], "" ); + break; + } + } + } + } + + if ( !set ) { + set = context.getElementsByTagName("*"); + } + + return {set: set, expr: expr}; +}; + +Sizzle.filter = function(expr, set, inplace, not){ + var old = expr, result = [], curLoop = set, match, anyFound, + isXMLFilter = set && set[0] && isXML(set[0]); + + while ( expr && set.length ) { + for ( var type in Expr.filter ) { + if ( (match = Expr.leftMatch[ type ].exec( expr )) != null && match[2] ) { + var filter = Expr.filter[ type ], found, item, left = match[1]; + anyFound = false; + + match.splice(1,1); + + if ( left.substr( left.length - 1 ) === "\\" ) { + continue; + } + + if ( curLoop === result ) { + result = []; + } + + if ( Expr.preFilter[ type ] ) { + match = Expr.preFilter[ type ]( match, curLoop, inplace, result, not, isXMLFilter ); + + if ( !match ) { + anyFound = found = true; + } else if ( match === true ) { + continue; + } + } + + if ( match ) { + for ( var i = 0; (item = curLoop[i]) != null; i++ ) { + if ( item ) { + found = filter( item, match, i, curLoop ); + var pass = not ^ !!found; + + if ( inplace && found != null ) { + if ( pass ) { + anyFound = true; + } else { + curLoop[i] = false; + } + } else if ( pass ) { + result.push( item ); + anyFound = true; + } + } + } + } + + if ( found !== undefined ) { + if ( !inplace ) { + curLoop = result; + } + + expr = expr.replace( Expr.match[ type ], "" ); + + if ( !anyFound ) { + return []; + } + + break; + } + } + } + + // Improper expression + if ( expr === old ) { + if ( anyFound == null ) { + Sizzle.error( expr ); + } else { + break; + } + } + + old = expr; + } + + return curLoop; +}; + +Sizzle.error = function( msg ) { + throw "Syntax error, unrecognized expression: " + msg; +}; + +var Expr = Sizzle.selectors = { + order: [ "ID", "NAME", "TAG" ], + match: { + ID: /#((?:[\w\u00c0-\uFFFF-]|\\.)+)/, + CLASS: /\.((?:[\w\u00c0-\uFFFF-]|\\.)+)/, + NAME: /\[name=['"]*((?:[\w\u00c0-\uFFFF-]|\\.)+)['"]*\]/, + ATTR: /\[\s*((?:[\w\u00c0-\uFFFF-]|\\.)+)\s*(?:(\S?=)\s*(['"]*)(.*?)\3|)\s*\]/, + TAG: /^((?:[\w\u00c0-\uFFFF\*-]|\\.)+)/, + CHILD: /:(only|nth|last|first)-child(?:\((even|odd|[\dn+-]*)\))?/, + POS: /:(nth|eq|gt|lt|first|last|even|odd)(?:\((\d*)\))?(?=[^-]|$)/, + PSEUDO: /:((?:[\w\u00c0-\uFFFF-]|\\.)+)(?:\((['"]?)((?:\([^\)]+\)|[^\(\)]*)+)\2\))?/ + }, + leftMatch: {}, + attrMap: { + "class": "className", + "for": "htmlFor" + }, + attrHandle: { + href: function(elem){ + return elem.getAttribute("href"); + } + }, + relative: { + "+": function(checkSet, part){ + var isPartStr = typeof part === "string", + isTag = isPartStr && !/\W/.test(part), + isPartStrNotTag = isPartStr && !isTag; + + if ( isTag ) { + part = part.toLowerCase(); + } + + for ( var i = 0, l = checkSet.length, elem; i < l; i++ ) { + if ( (elem = checkSet[i]) ) { + while ( (elem = elem.previousSibling) && elem.nodeType !== 1 ) {} + + checkSet[i] = isPartStrNotTag || elem && elem.nodeName.toLowerCase() === part ? + elem || false : + elem === part; + } + } + + if ( isPartStrNotTag ) { + Sizzle.filter( part, checkSet, true ); + } + }, + ">": function(checkSet, part){ + var isPartStr = typeof part === "string"; + + if ( isPartStr && !/\W/.test(part) ) { + part = part.toLowerCase(); + + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + if ( elem ) { + var parent = elem.parentNode; + checkSet[i] = parent.nodeName.toLowerCase() === part ? parent : false; + } + } + } else { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + if ( elem ) { + checkSet[i] = isPartStr ? + elem.parentNode : + elem.parentNode === part; + } + } + + if ( isPartStr ) { + Sizzle.filter( part, checkSet, true ); + } + } + }, + "": function(checkSet, part, isXML){ + var doneName = done++, checkFn = dirCheck; + + if ( typeof part === "string" && !/\W/.test(part) ) { + var nodeCheck = part = part.toLowerCase(); + checkFn = dirNodeCheck; + } + + checkFn("parentNode", part, doneName, checkSet, nodeCheck, isXML); + }, + "~": function(checkSet, part, isXML){ + var doneName = done++, checkFn = dirCheck; + + if ( typeof part === "string" && !/\W/.test(part) ) { + var nodeCheck = part = part.toLowerCase(); + checkFn = dirNodeCheck; + } + + checkFn("previousSibling", part, doneName, checkSet, nodeCheck, isXML); + } + }, + find: { + ID: function(match, context, isXML){ + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + return m ? [m] : []; + } + }, + NAME: function(match, context){ + if ( typeof context.getElementsByName !== "undefined" ) { + var ret = [], results = context.getElementsByName(match[1]); + + for ( var i = 0, l = results.length; i < l; i++ ) { + if ( results[i].getAttribute("name") === match[1] ) { + ret.push( results[i] ); + } + } + + return ret.length === 0 ? null : ret; + } + }, + TAG: function(match, context){ + return context.getElementsByTagName(match[1]); + } + }, + preFilter: { + CLASS: function(match, curLoop, inplace, result, not, isXML){ + match = " " + match[1].replace(/\\/g, "") + " "; + + if ( isXML ) { + return match; + } + + for ( var i = 0, elem; (elem = curLoop[i]) != null; i++ ) { + if ( elem ) { + if ( not ^ (elem.className && (" " + elem.className + " ").replace(/[\t\n]/g, " ").indexOf(match) >= 0) ) { + if ( !inplace ) { + result.push( elem ); + } + } else if ( inplace ) { + curLoop[i] = false; + } + } + } + + return false; + }, + ID: function(match){ + return match[1].replace(/\\/g, ""); + }, + TAG: function(match, curLoop){ + return match[1].toLowerCase(); + }, + CHILD: function(match){ + if ( match[1] === "nth" ) { + // parse equations like 'even', 'odd', '5', '2n', '3n+2', '4n-1', '-n+6' + var test = /(-?)(\d*)n((?:\+|-)?\d*)/.exec( + match[2] === "even" && "2n" || match[2] === "odd" && "2n+1" || + !/\D/.test( match[2] ) && "0n+" + match[2] || match[2]); + + // calculate the numbers (first)n+(last) including if they are negative + match[2] = (test[1] + (test[2] || 1)) - 0; + match[3] = test[3] - 0; + } + + // TODO: Move to normal caching system + match[0] = done++; + + return match; + }, + ATTR: function(match, curLoop, inplace, result, not, isXML){ + var name = match[1].replace(/\\/g, ""); + + if ( !isXML && Expr.attrMap[name] ) { + match[1] = Expr.attrMap[name]; + } + + if ( match[2] === "~=" ) { + match[4] = " " + match[4] + " "; + } + + return match; + }, + PSEUDO: function(match, curLoop, inplace, result, not){ + if ( match[1] === "not" ) { + // If we're dealing with a complex expression, or a simple one + if ( ( chunker.exec(match[3]) || "" ).length > 1 || /^\w/.test(match[3]) ) { + match[3] = Sizzle(match[3], null, null, curLoop); + } else { + var ret = Sizzle.filter(match[3], curLoop, inplace, true ^ not); + if ( !inplace ) { + result.push.apply( result, ret ); + } + return false; + } + } else if ( Expr.match.POS.test( match[0] ) || Expr.match.CHILD.test( match[0] ) ) { + return true; + } + + return match; + }, + POS: function(match){ + match.unshift( true ); + return match; + } + }, + filters: { + enabled: function(elem){ + return elem.disabled === false && elem.type !== "hidden"; + }, + disabled: function(elem){ + return elem.disabled === true; + }, + checked: function(elem){ + return elem.checked === true; + }, + selected: function(elem){ + // Accessing this property makes selected-by-default + // options in Safari work properly + elem.parentNode.selectedIndex; + return elem.selected === true; + }, + parent: function(elem){ + return !!elem.firstChild; + }, + empty: function(elem){ + return !elem.firstChild; + }, + has: function(elem, i, match){ + return !!Sizzle( match[3], elem ).length; + }, + header: function(elem){ + return /h\d/i.test( elem.nodeName ); + }, + text: function(elem){ + return "text" === elem.type; + }, + radio: function(elem){ + return "radio" === elem.type; + }, + checkbox: function(elem){ + return "checkbox" === elem.type; + }, + file: function(elem){ + return "file" === elem.type; + }, + password: function(elem){ + return "password" === elem.type; + }, + submit: function(elem){ + return "submit" === elem.type; + }, + image: function(elem){ + return "image" === elem.type; + }, + reset: function(elem){ + return "reset" === elem.type; + }, + button: function(elem){ + return "button" === elem.type || elem.nodeName.toLowerCase() === "button"; + }, + input: function(elem){ + return /input|select|textarea|button/i.test(elem.nodeName); + } + }, + setFilters: { + first: function(elem, i){ + return i === 0; + }, + last: function(elem, i, match, array){ + return i === array.length - 1; + }, + even: function(elem, i){ + return i % 2 === 0; + }, + odd: function(elem, i){ + return i % 2 === 1; + }, + lt: function(elem, i, match){ + return i < match[3] - 0; + }, + gt: function(elem, i, match){ + return i > match[3] - 0; + }, + nth: function(elem, i, match){ + return match[3] - 0 === i; + }, + eq: function(elem, i, match){ + return match[3] - 0 === i; + } + }, + filter: { + PSEUDO: function(elem, match, i, array){ + var name = match[1], filter = Expr.filters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + } else if ( name === "contains" ) { + return (elem.textContent || elem.innerText || getText([ elem ]) || "").indexOf(match[3]) >= 0; + } else if ( name === "not" ) { + var not = match[3]; + + for ( var i = 0, l = not.length; i < l; i++ ) { + if ( not[i] === elem ) { + return false; + } + } + + return true; + } else { + Sizzle.error( "Syntax error, unrecognized expression: " + name ); + } + }, + CHILD: function(elem, match){ + var type = match[1], node = elem; + switch (type) { + case 'only': + case 'first': + while ( (node = node.previousSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + if ( type === "first" ) { + return true; + } + node = elem; + case 'last': + while ( (node = node.nextSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + return true; + case 'nth': + var first = match[2], last = match[3]; + + if ( first === 1 && last === 0 ) { + return true; + } + + var doneName = match[0], + parent = elem.parentNode; + + if ( parent && (parent.sizcache !== doneName || !elem.nodeIndex) ) { + var count = 0; + for ( node = parent.firstChild; node; node = node.nextSibling ) { + if ( node.nodeType === 1 ) { + node.nodeIndex = ++count; + } + } + parent.sizcache = doneName; + } + + var diff = elem.nodeIndex - last; + if ( first === 0 ) { + return diff === 0; + } else { + return ( diff % first === 0 && diff / first >= 0 ); + } + } + }, + ID: function(elem, match){ + return elem.nodeType === 1 && elem.getAttribute("id") === match; + }, + TAG: function(elem, match){ + return (match === "*" && elem.nodeType === 1) || elem.nodeName.toLowerCase() === match; + }, + CLASS: function(elem, match){ + return (" " + (elem.className || elem.getAttribute("class")) + " ") + .indexOf( match ) > -1; + }, + ATTR: function(elem, match){ + var name = match[1], + result = Expr.attrHandle[ name ] ? + Expr.attrHandle[ name ]( elem ) : + elem[ name ] != null ? + elem[ name ] : + elem.getAttribute( name ), + value = result + "", + type = match[2], + check = match[4]; + + return result == null ? + type === "!=" : + type === "=" ? + value === check : + type === "*=" ? + value.indexOf(check) >= 0 : + type === "~=" ? + (" " + value + " ").indexOf(check) >= 0 : + !check ? + value && result !== false : + type === "!=" ? + value !== check : + type === "^=" ? + value.indexOf(check) === 0 : + type === "$=" ? + value.substr(value.length - check.length) === check : + type === "|=" ? + value === check || value.substr(0, check.length + 1) === check + "-" : + false; + }, + POS: function(elem, match, i, array){ + var name = match[2], filter = Expr.setFilters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + } + } + } +}; + +var origPOS = Expr.match.POS; + +for ( var type in Expr.match ) { + Expr.match[ type ] = new RegExp( Expr.match[ type ].source + /(?![^\[]*\])(?![^\(]*\))/.source ); + Expr.leftMatch[ type ] = new RegExp( /(^(?:.|\r|\n)*?)/.source + Expr.match[ type ].source.replace(/\\(\d+)/g, function(all, num){ + return "\\" + (num - 0 + 1); + })); +} + +var makeArray = function(array, results) { + array = Array.prototype.slice.call( array, 0 ); + + if ( results ) { + results.push.apply( results, array ); + return results; + } + + return array; +}; + +// Perform a simple check to determine if the browser is capable of +// converting a NodeList to an array using builtin methods. +// Also verifies that the returned array holds DOM nodes +// (which is not the case in the Blackberry browser) +try { + Array.prototype.slice.call( document.documentElement.childNodes, 0 )[0].nodeType; + +// Provide a fallback method if it does not work +} catch(e){ + makeArray = function(array, results) { + var ret = results || []; + + if ( toString.call(array) === "[object Array]" ) { + Array.prototype.push.apply( ret, array ); + } else { + if ( typeof array.length === "number" ) { + for ( var i = 0, l = array.length; i < l; i++ ) { + ret.push( array[i] ); + } + } else { + for ( var i = 0; array[i]; i++ ) { + ret.push( array[i] ); + } + } + } + + return ret; + }; +} + +var sortOrder; + +if ( document.documentElement.compareDocumentPosition ) { + sortOrder = function( a, b ) { + if ( !a.compareDocumentPosition || !b.compareDocumentPosition ) { + if ( a == b ) { + hasDuplicate = true; + } + return a.compareDocumentPosition ? -1 : 1; + } + + var ret = a.compareDocumentPosition(b) & 4 ? -1 : a === b ? 0 : 1; + if ( ret === 0 ) { + hasDuplicate = true; + } + return ret; + }; +} else if ( "sourceIndex" in document.documentElement ) { + sortOrder = function( a, b ) { + if ( !a.sourceIndex || !b.sourceIndex ) { + if ( a == b ) { + hasDuplicate = true; + } + return a.sourceIndex ? -1 : 1; + } + + var ret = a.sourceIndex - b.sourceIndex; + if ( ret === 0 ) { + hasDuplicate = true; + } + return ret; + }; +} else if ( document.createRange ) { + sortOrder = function( a, b ) { + if ( !a.ownerDocument || !b.ownerDocument ) { + if ( a == b ) { + hasDuplicate = true; + } + return a.ownerDocument ? -1 : 1; + } + + var aRange = a.ownerDocument.createRange(), bRange = b.ownerDocument.createRange(); + aRange.setStart(a, 0); + aRange.setEnd(a, 0); + bRange.setStart(b, 0); + bRange.setEnd(b, 0); + var ret = aRange.compareBoundaryPoints(Range.START_TO_END, bRange); + if ( ret === 0 ) { + hasDuplicate = true; + } + return ret; + }; +} + +// Utility function for retreiving the text value of an array of DOM nodes +function getText( elems ) { + var ret = "", elem; + + for ( var i = 0; elems[i]; i++ ) { + elem = elems[i]; + + // Get the text from text nodes and CDATA nodes + if ( elem.nodeType === 3 || elem.nodeType === 4 ) { + ret += elem.nodeValue; + + // Traverse everything else, except comment nodes + } else if ( elem.nodeType !== 8 ) { + ret += getText( elem.childNodes ); + } + } + + return ret; +} + +// Check to see if the browser returns elements by name when +// querying by getElementById (and provide a workaround) +(function(){ + // We're going to inject a fake input element with a specified name + var form = document.createElement("div"), + id = "script" + (new Date).getTime(); + form.innerHTML = ""; + + // Inject it into the root element, check its status, and remove it quickly + var root = document.documentElement; + root.insertBefore( form, root.firstChild ); + + // The workaround has to do additional checks after a getElementById + // Which slows things down for other browsers (hence the branching) + if ( document.getElementById( id ) ) { + Expr.find.ID = function(match, context, isXML){ + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + return m ? m.id === match[1] || typeof m.getAttributeNode !== "undefined" && m.getAttributeNode("id").nodeValue === match[1] ? [m] : undefined : []; + } + }; + + Expr.filter.ID = function(elem, match){ + var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode("id"); + return elem.nodeType === 1 && node && node.nodeValue === match; + }; + } + + root.removeChild( form ); + root = form = null; // release memory in IE +})(); + +(function(){ + // Check to see if the browser returns only elements + // when doing getElementsByTagName("*") + + // Create a fake element + var div = document.createElement("div"); + div.appendChild( document.createComment("") ); + + // Make sure no comments are found + if ( div.getElementsByTagName("*").length > 0 ) { + Expr.find.TAG = function(match, context){ + var results = context.getElementsByTagName(match[1]); + + // Filter out possible comments + if ( match[1] === "*" ) { + var tmp = []; + + for ( var i = 0; results[i]; i++ ) { + if ( results[i].nodeType === 1 ) { + tmp.push( results[i] ); + } + } + + results = tmp; + } + + return results; + }; + } + + // Check to see if an attribute returns normalized href attributes + div.innerHTML = ""; + if ( div.firstChild && typeof div.firstChild.getAttribute !== "undefined" && + div.firstChild.getAttribute("href") !== "#" ) { + Expr.attrHandle.href = function(elem){ + return elem.getAttribute("href", 2); + }; + } + + div = null; // release memory in IE +})(); + +if ( document.querySelectorAll ) { + (function(){ + var oldSizzle = Sizzle, div = document.createElement("div"); + div.innerHTML = "

"; + + // Safari can't handle uppercase or unicode characters when + // in quirks mode. + if ( div.querySelectorAll && div.querySelectorAll(".TEST").length === 0 ) { + return; + } + + Sizzle = function(query, context, extra, seed){ + context = context || document; + + // Only use querySelectorAll on non-XML documents + // (ID selectors don't work in non-HTML documents) + if ( !seed && context.nodeType === 9 && !isXML(context) ) { + try { + return makeArray( context.querySelectorAll(query), extra ); + } catch(e){} + } + + return oldSizzle(query, context, extra, seed); + }; + + for ( var prop in oldSizzle ) { + Sizzle[ prop ] = oldSizzle[ prop ]; + } + + div = null; // release memory in IE + })(); +} + +(function(){ + var div = document.createElement("div"); + + div.innerHTML = "
"; + + // Opera can't find a second classname (in 9.6) + // Also, make sure that getElementsByClassName actually exists + if ( !div.getElementsByClassName || div.getElementsByClassName("e").length === 0 ) { + return; + } + + // Safari caches class attributes, doesn't catch changes (in 3.2) + div.lastChild.className = "e"; + + if ( div.getElementsByClassName("e").length === 1 ) { + return; + } + + Expr.order.splice(1, 0, "CLASS"); + Expr.find.CLASS = function(match, context, isXML) { + if ( typeof context.getElementsByClassName !== "undefined" && !isXML ) { + return context.getElementsByClassName(match[1]); + } + }; + + div = null; // release memory in IE +})(); + +function dirNodeCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + if ( elem ) { + elem = elem[dir]; + var match = false; + + while ( elem ) { + if ( elem.sizcache === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 && !isXML ){ + elem.sizcache = doneName; + elem.sizset = i; + } + + if ( elem.nodeName.toLowerCase() === cur ) { + match = elem; + break; + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +function dirCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + if ( elem ) { + elem = elem[dir]; + var match = false; + + while ( elem ) { + if ( elem.sizcache === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 ) { + if ( !isXML ) { + elem.sizcache = doneName; + elem.sizset = i; + } + if ( typeof cur !== "string" ) { + if ( elem === cur ) { + match = true; + break; + } + + } else if ( Sizzle.filter( cur, [elem] ).length > 0 ) { + match = elem; + break; + } + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +var contains = document.compareDocumentPosition ? function(a, b){ + return !!(a.compareDocumentPosition(b) & 16); +} : function(a, b){ + return a !== b && (a.contains ? a.contains(b) : true); +}; + +var isXML = function(elem){ + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = (elem ? elem.ownerDocument || elem : 0).documentElement; + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +var posProcess = function(selector, context){ + var tmpSet = [], later = "", match, + root = context.nodeType ? [context] : context; + + // Position selectors must be done after the filter + // And so must :not(positional) so we move all PSEUDOs to the end + while ( (match = Expr.match.PSEUDO.exec( selector )) ) { + later += match[0]; + selector = selector.replace( Expr.match.PSEUDO, "" ); + } + + selector = Expr.relative[selector] ? selector + "*" : selector; + + for ( var i = 0, l = root.length; i < l; i++ ) { + Sizzle( selector, root[i], tmpSet ); + } + + return Sizzle.filter( later, tmpSet ); +}; + +// EXPOSE +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.filters; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = getText; +jQuery.isXMLDoc = isXML; +jQuery.contains = contains; + +return; + +window.Sizzle = Sizzle; + +})(); +var runtil = /Until$/, + rparentsprev = /^(?:parents|prevUntil|prevAll)/, + // Note: This RegExp should be improved, or likely pulled from Sizzle + rmultiselector = /,/, + slice = Array.prototype.slice; + +// Implement the identical functionality for filter and not +var winnow = function( elements, qualifier, keep ) { + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep(elements, function( elem, i ) { + return !!qualifier.call( elem, i, elem ) === keep; + }); + + } else if ( qualifier.nodeType ) { + return jQuery.grep(elements, function( elem, i ) { + return (elem === qualifier) === keep; + }); + + } else if ( typeof qualifier === "string" ) { + var filtered = jQuery.grep(elements, function( elem ) { + return elem.nodeType === 1; + }); + + if ( isSimple.test( qualifier ) ) { + return jQuery.filter(qualifier, filtered, !keep); + } else { + qualifier = jQuery.filter( qualifier, filtered ); + } + } + + return jQuery.grep(elements, function( elem, i ) { + return (jQuery.inArray( elem, qualifier ) >= 0) === keep; + }); +}; + +jQuery.fn.extend({ + find: function( selector ) { + var ret = this.pushStack( "", "find", selector ), length = 0; + + for ( var i = 0, l = this.length; i < l; i++ ) { + length = ret.length; + jQuery.find( selector, this[i], ret ); + + if ( i > 0 ) { + // Make sure that the results are unique + for ( var n = length; n < ret.length; n++ ) { + for ( var r = 0; r < length; r++ ) { + if ( ret[r] === ret[n] ) { + ret.splice(n--, 1); + break; + } + } + } + } + } + + return ret; + }, + + has: function( target ) { + var targets = jQuery( target ); + return this.filter(function() { + for ( var i = 0, l = targets.length; i < l; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + not: function( selector ) { + return this.pushStack( winnow(this, selector, false), "not", selector); + }, + + filter: function( selector ) { + return this.pushStack( winnow(this, selector, true), "filter", selector ); + }, + + is: function( selector ) { + return !!selector && jQuery.filter( selector, this ).length > 0; + }, + + closest: function( selectors, context ) { + if ( jQuery.isArray( selectors ) ) { + var ret = [], cur = this[0], match, matches = {}, selector; + + if ( cur && selectors.length ) { + for ( var i = 0, l = selectors.length; i < l; i++ ) { + selector = selectors[i]; + + if ( !matches[selector] ) { + matches[selector] = jQuery.expr.match.POS.test( selector ) ? + jQuery( selector, context || this.context ) : + selector; + } + } + + while ( cur && cur.ownerDocument && cur !== context ) { + for ( selector in matches ) { + match = matches[selector]; + + if ( match.jquery ? match.index(cur) > -1 : jQuery(cur).is(match) ) { + ret.push({ selector: selector, elem: cur }); + delete matches[selector]; + } + } + cur = cur.parentNode; + } + } + + return ret; + } + + var pos = jQuery.expr.match.POS.test( selectors ) ? + jQuery( selectors, context || this.context ) : null; + + return this.map(function( i, cur ) { + while ( cur && cur.ownerDocument && cur !== context ) { + if ( pos ? pos.index(cur) > -1 : jQuery(cur).is(selectors) ) { + return cur; + } + cur = cur.parentNode; + } + return null; + }); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + if ( !elem || typeof elem === "string" ) { + return jQuery.inArray( this[0], + // If it receives a string, the selector is used + // If it receives nothing, the siblings are used + elem ? jQuery( elem ) : this.parent().children() ); + } + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + var set = typeof selector === "string" ? + jQuery( selector, context || this.context ) : + jQuery.makeArray( selector ), + all = jQuery.merge( this.get(), set ); + + return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ? + all : + jQuery.unique( all ) ); + }, + + andSelf: function() { + return this.add( this.prevObject ); + } +}); + +// A painfully simple check to see if an element is disconnected +// from a document (should be improved, where feasible). +function isDisconnected( node ) { + return !node || !node.parentNode || node.parentNode.nodeType === 11; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return jQuery.nth( elem, 2, "nextSibling" ); + }, + prev: function( elem ) { + return jQuery.nth( elem, 2, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( elem.parentNode.firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.makeArray( elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ); + + if ( !runtil.test( name ) ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + ret = this.length > 1 ? jQuery.unique( ret ) : ret; + + if ( (this.length > 1 || rmultiselector.test( selector )) && rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + + return this.pushStack( ret, name, slice.call(arguments).join(",") ); + }; +}); + +jQuery.extend({ + filter: function( expr, elems, not ) { + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return jQuery.find.matches(expr, elems); + }, + + dir: function( elem, dir, until ) { + var matched = [], cur = elem[dir]; + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + nth: function( cur, result, dir, elem ) { + result = result || 1; + var num = 0; + + for ( ; cur; cur = cur[dir] ) { + if ( cur.nodeType === 1 && ++num === result ) { + break; + } + } + + return cur; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); +var rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g, + rleadingWhitespace = /^\s+/, + rxhtmlTag = /(<([\w:]+)[^>]*?)\/>/g, + rselfClosing = /^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i, + rtagName = /<([\w:]+)/, + rtbody = /"; + }, + wrapMap = { + option: [ 1, "" ], + legend: [ 1, "
", "
" ], + thead: [ 1, "", "
" ], + tr: [ 2, "", "
" ], + td: [ 3, "", "
" ], + col: [ 2, "", "
" ], + area: [ 1, "", "" ], + _default: [ 0, "", "" ] + }; + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +// IE can't serialize and + + + + + + + + + +
+
+
+ +
+
+
+ {% for counter,label in enumerate(schemes) %} + {% if label == scheme %} +
+ {% else %} +
+ {% end %} + {{label}} +
+ {% end %} +
+ +
+ + {% for dict in errors %} +
+ +
+
+ +
+
+ {{ dict['testname'] }} +
+
+ {{ dict['status'] }} +
+ +
+ + {% if summary[ dict['testname'] ]['alert'] %} + Warning: Check for screenshot mismatch + {% end %} + +
+ +
+
+ +
+
+
+ Name: {{ dict['testname'] }}
+ Status: {{ dict['status'] }}
+ Timestamp: {{ dict['timestamp'] }}
+ {% for counter, image in enumerate(summary[ dict['testname'] ]['images']) %} +
+ {{ image[:-4].replace("_"," ").title() }}: +
+
+
+
+ Baseline
+ +
+
+ Current
+ +
+
+
+ Actions
+ + + {% if summary[ dict['testname'] ]['warnings'][image] %} + {{ summary[ dict['testname'] ]['warnings'][image] }} + {% end %} + + +
+ {% if summary[ dict['testname'] ]['warnings'][image] is not None %} + + + {% else %} + No actions available. + {% end %} +
+ +
+
+ {% end %} + +
+
+ Error Details +
+ 
+{{ summary[dict['testname']]['details'] }} 
+
+
+
+ +
+ {% end %} + {% for dict in passes %} +
+
+
+ +
+
+ {{ dict['testname'] }} +
+
+ {{ dict['status'] }} +
+ +
+ + {% if summary[ dict['testname'] ]['alert'] is True %} + Warning: Check for screenshot mismatch + {% end %} + +
+ +
+
+ +
+
+
+ Name: {{ dict['testname'] }}
+ Status: {{ dict['status'] }}
+ Timestamp: {{ dict['timestamp'] }}
+ + + {% for counter, image in enumerate(summary[dict['testname']]['images']) %} +
+ {{ image[:-4].replace("_"," ").title() }}: +
+ +
+
+
+ Baseline
+ +
+
+ Current
+ +
+
+
+ Actions
+ + + {% if summary[ dict['testname'] ]['warnings'][image] is not None %} + {{ summary[ dict['testname'] ]['warnings'][image] }} + {% end %} + + +
+ {% if summary[ dict['testname'] ]['warnings'][image] is not None %} + + {% else %} + No actions available. + {% end %} + +
+
+
+ + {% end %} +
+ +
+ {% end %} + +
+
+
+ + \ No newline at end of file diff --git a/clients/ios/testing/tests/testContacts.js b/clients/ios/testing/tests/testContacts.js new file mode 100644 index 0000000..13d11eb --- /dev/null +++ b/clients/ios/testing/tests/testContacts.js @@ -0,0 +1,73 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, import contacts + * @author: Greg Vandenberg + * + */ + +VFRunTest("testContacts", runStateEnum.DASHBOARD, function(testname, util, log) { + + var t = TEST_USERS[0]; + // check if you are on dashboard + var dash = util.gotoDashboard(); + + // add a contact + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + log.debug("------------------------------------"); + log.debug("Import Gmail contacts"); + log.debug("------------------------------------"); + contacts.importGmailContacts(); + target.delay(2); // for accurate screen shot + util.screenCapture("at_import_gmail_contacts"); + + log.debug("------------------------------------"); + log.debug("Import Facebook friends"); + log.debug("------------------------------------"); + contacts.importFacebookFriends(); + util.screenCapture("at_import_facebook_contacts"); + + contacts.selectBackNav(); + + log.debug("------------------------------------"); + log.debug("Search for name in All contacts"); + log.debug("------------------------------------"); + util.dismissAlert("View All"); + target.delay(1); // TODO: fix this + contacts.selectButtonAll(); + contacts.enterSearchTerm(t.firstname); + + util.screenCapture("contacts_search_form"); + + log.debug("------------------------------------"); + log.debug("Send email to contact"); + log.debug("------------------------------------"); + contacts.selectContact(1); + contacts.selectBackNav(); + contacts.selectBackNav(); + + log.debug("------------------------------------"); + log.debug("Find a contact"); + log.debug("------------------------------------"); + contacts.selectAddContacts(); + + contacts.setContactEmail('asdf'); + + util.dismissAlert("OK"); + contacts.setContactEmail(t.email); + + util.screenCapture("contacts_dashboard"); + + // TODO: Search for VF contact + // TODO: Search for ALL contact + // TODO: Use right jump scroll 'w' + + +}); diff --git a/clients/ios/testing/tests/testConversations.js b/clients/ios/testing/tests/testConversations.js new file mode 100644 index 0000000..e3ca86f --- /dev/null +++ b/clients/ios/testing/tests/testConversations.js @@ -0,0 +1,83 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, start a conversation + * @author: Greg Vandenberg + * + */ + +VFRunTest("testConversations", runStateEnum.DASHBOARD, function(testname, util, log) { + var dash = util.gotoDashboard(); + var nav = util.getNav(); + var user = new VFUser(); + var t = TEST_USERS[1]; + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + + // goto conversation + var convo = dash.gotoConversations(); + + log.debug("------------------------------------"); + log.debug("Select Action with zero conversations."); + log.debug("------------------------------------"); + convo.selectActionButton(); + util.screenCapture('select_action_popup'); + util.dismissAlert("Ok"); + + log.debug("------------------------------------"); + log.debug("Create a new conversation."); + log.debug("------------------------------------"); + // compose new convo + convo.selectCompose(); + + log.debug("------------------------------------"); + log.debug("Add people to conversation."); + log.debug("------------------------------------"); + // add people to convo + convo.selectAddPeople(); + convo.setPerson(t.firstname + " " + t.lastname + "\n"); + + //set title + convo.setTitle('Test Conversation'); + + util.screenCapture('set_person_title'); + + //Work around for issue #415 + convo.selectAddPeople(); + + log.debug("------------------------------------"); + log.debug("Add photo to conversation."); + log.debug("------------------------------------"); + // add photo to convo + convo.selectAddPhotos(); + + convo.selectNumImages(1); + + convo.selectAddPhotosNav(); + + convo.selectStart(); + + util.screenCapture('convo_started'); + + log.debug("------------------------------------"); + log.debug("Add comment to conversation."); + log.debug("------------------------------------"); + convo.addComment('test comment'); + util.screenCapture('comment_added'); + + convo.selectSend(); + convo.selectBack(); + util.gotoDashboard(); + util.screenCapture('dashboard_1_convo'); + +}); diff --git a/clients/ios/testing/tests/testExportPhotos.js b/clients/ios/testing/tests/testExportPhotos.js new file mode 100644 index 0000000..f47879a --- /dev/null +++ b/clients/ios/testing/tests/testExportPhotos.js @@ -0,0 +1,64 @@ +/** + * + * Copyright 2013 Viewfinder Inc. All Rights Reserved. + * Description: Export photos, error on none selected starting from the dashboard. + * Author: Greg Vandenberg + * + */ + +VFRunTest("testExportPhotos", runStateEnum.DASHBOARD, function(testname, util, log) { + + var dash = new VFDashboard(); + var user = new VFUser(); + var t = TEST_USERS[1]; + + log.debug("------------------------------------"); + log.debug("Go to Library."); + log.debug("------------------------------------"); + var library = dash.gotoLibrary(); + target.delay(2); // for accurate screen shot + util.screenCapture('personal_library'); + + log.debug("------------------------------------"); + log.debug("Export 5 photos to camera roll."); + log.debug("------------------------------------"); + library.selectActionButton(); + target.delay(2); // for accurate screen shot + util.screenCapture('library_action'); + + library.selectNumImages(5); + + library.selectExportButton(); + target.delay(2); // for accurate screen shot + util.screenCapture('export_button_5_photos'); + + library.selectConfirmExport(); + + log.debug("------------------------------------"); + log.debug("Go back to dashboard."); + log.debug("------------------------------------"); + util.gotoDashboard(); + util.screenCapture('dashboard'); + + log.debug("------------------------------------"); + log.debug("Go to library."); + log.debug("------------------------------------"); + library = dash.gotoLibrary(); + + log.debug("------------------------------------"); + log.debug("Export 1 photo to camera roll."); + log.debug("------------------------------------"); + library.selectActionButton(); + library.selectNumImages(1); + + library.selectExportButton(); + util.screenCapture('export_button_1_photo'); + + library.selectConfirmExport(); + + log.debug("------------------------------------"); + log.debug("Go back to dashboard."); + log.debug("------------------------------------"); + util.gotoDashboard(); + +}); diff --git a/clients/ios/testing/tests/testMyInfo.js b/clients/ios/testing/tests/testMyInfo.js new file mode 100644 index 0000000..03264cb --- /dev/null +++ b/clients/ios/testing/tests/testMyInfo.js @@ -0,0 +1,72 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From logged in on the dashboard, merge and link + * identities to your account + * @author: Greg Vandenberg + * + */ + +VFRunTest("testMyInfo", runStateEnum.DASHBOARD, function(testname, util, log) { + + var t = TEST_USERS[0]; + var t5 = TEST_USERS[4]; + var user = new VFUser(); + var dash = util.gotoDashboard(); + var info = dash.gotoMyInfo(); + + info.selectAddEmailOrMobile(); + + log.debug("------------------------------------"); + log.debug("MyInfo: Incorrect email format."); + log.debug("------------------------------------"); + info.setEmailAddress('tester'); + + info.selectAdd(); + target.delay(2); // TODO: poll instead + util.screenCapture('email_is_incorrect_format'); + util.dismissAlert("Let me fix that..."); + + log.debug("------------------------------------"); + log.debug("MyInfo: Email already linked."); + log.debug("------------------------------------"); + info.setEmailAddress(t.email); + + info.selectAdd(); + + util.screenCapture('email_is_already_linked'); + util.dismissAlert("OK"); + + log.debug("------------------------------------"); + log.debug("MyInfo: Link email."); + log.debug("------------------------------------"); + info.setEmailAddress(t5.email); + + info.selectAdd(); + //util.delay(2); // TODO: retry in get_access_code method + var access_code = user.get_access_code('Email:' + t5.email); + util.delay(2); + info.setAccessCode(access_code); + + info.selectContinue(); + util.delay(2); // wait for merge ui to settle + util.screenCapture('merged_or_linked_account'); + + info.selectAddEmailOrMobile(); + + log.debug("------------------------------------"); + log.debug("MyInfo: Link mobile number."); + log.debug("------------------------------------"); + info.setMobileNumber(t5.mobile); + + info.selectAdd(); + //util.delay(2); // TODO: retry in get_access_code method + var access_code2 = user.get_access_code('Phone:' + t5.mobile); + + info.setAccessCode(access_code2); + + info.selectContinue(); + util.delay(2); // wait for merge ui to settle + util.screenCapture('merged_or_linked_number'); + +}); diff --git a/clients/ios/testing/tests/testOnboardingLoginExistingUser.js b/clients/ios/testing/tests/testOnboardingLoginExistingUser.js new file mode 100644 index 0000000..e678f65 --- /dev/null +++ b/clients/ios/testing/tests/testOnboardingLoginExistingUser.js @@ -0,0 +1,89 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, login with new user (terminated) and + * an existing user + * @author: Greg Vandenberg + * + */ + +VFRunTest("testOnboardingLoginExistingUser", runStateEnum.ONBOARDING, function(testname, util, log) { + + var ob = new VFOnboarding(util.getNav()); + var t = TEST_USERS[0]; + var user = new VFUser(); + user.register(t); + + log.debug("------------------------------------"); + log.debug("Start Login process (invalid email)."); + log.debug("------------------------------------"); + ob.gotoLogin(); + util.screenCapture('login_form'); + ob.login().setInvalidEmailFormat(); + ob.login().setPassword(t.password); + target.delay(2); // for accurate screen shot + util.screenCapture('login_form_filled_out'); + ob.login().selectLogin(); + + util.screenCapture('login_invalid_email_error'); + util.dismissAlert("Let me fix that..."); + + log.debug("------------------------------------"); + log.debug("Login with invalid account."); + log.debug("------------------------------------"); + ob.login().setValidEmailFormat('invalid@email.com'); + ob.login().setPassword(t.password); + ob.login().selectLogin(); + util.screenCapture('login_invalid_user_error'); + util.dismissAlert("OK"); + ob.login().selectCancel(); + + log.debug("------------------------------------"); + log.debug("Login with 3 character password."); + log.debug("------------------------------------"); + ob.gotoLogin(); + ob.login().setValidEmailFormat(t.email); + ob.login().setPassword('foo'); + ob.login().selectLogin(); + util.screenCapture('login_incorrect_password_error'); + util.dismissAlert("OK"); + ob.login().selectCancel(); + + log.debug("------------------------------------"); + log.debug("Forgot password: clear email and password text if necessary"); + log.debug("------------------------------------"); + ob.gotoLogin(); + ob.login().setPassword(''); + ob.login().setValidEmailFormat(''); + ob.login().selectForgotPassword(); + + log.debug("------------------------------------"); + log.debug("Forgot password: Select 'Back'. Set valid email"); + log.debug("------------------------------------"); + ob.login().selectBack(); + ob.login().setValidEmailFormat(t.email); + ob.login().selectForgotPassword(); + ob.login().selectSubmit(); + + log.debug("------------------------------------"); + log.debug("Forgot password: Get Access Code... Continue."); + log.debug("------------------------------------"); + util.screenCapture('login_set_access_code'); + var access_code = user.get_access_code('Email:'+t.email); + ob.signup().setAccessCode(access_code); + ob.signup().selectContinue(); + + log.debug("------------------------------------"); + log.debug("Forgot password: Set new password (currently broken by issue #493)"); + log.debug("------------------------------------"); + ob.login().setNewPassword(t.password); + ob.login().setConfirmPassword(t.password); + util.screenCapture('login_confirm_password'); + ob.login().selectSubmit(); + + target.delay(2); // for accurate screen shot + util.screenCapture('logged_in_dashboard2'); + + + +}); diff --git a/clients/ios/testing/tests/testOnboardingSignUp.js b/clients/ios/testing/tests/testOnboardingSignUp.js new file mode 100644 index 0000000..54b9fb7 --- /dev/null +++ b/clients/ios/testing/tests/testOnboardingSignUp.js @@ -0,0 +1,107 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, sign up with new user (terminated) and + * an existing user + * @author: Greg Vandenberg + * + */ + +VFRunTest("testOnboardingSignUp", runStateEnum.ONBOARDING, function(testname, util, log) { + + var ob = new VFOnboarding(util.getNav()); + // TODO(ben): the flick gesture doesn't seem to be working. + /*log.debug("------------------------------------"); + log.debug("Explore intro screens."); + log.debug("------------------------------------"); + assertTrue(ob.gotoIntroDashboard(),"At intro dashboard."); + util.screenCapture("At_intro_dashboard"); + assertTrue(ob.gotoIntroLibrary(),"At intro library."); + util.screenCapture("At_intro_library"); + assertTrue(ob.gotoIntroConversation(),"At intro inbox."); + util.screenCapture("At_intro_inbox");*/ + + log.debug("------------------------------------"); + log.debug("Start Sign Up process (invalid user)."); + log.debug("------------------------------------"); + + var nav = util.getNav(); + var user = new VFUser(); + + assertTrue(ob.gotoSignup(),"At signup form."); + + var t = TEST_USERS[0]; + ob.signup().setFirstName(t.firstname); + ob.signup().setLastName(t.lastname); + ob.signup().setInvalidEmailFormat(); + ob.signup().setPassword(t.password); + ob.signup().selectCreateAccount(); + util.screenCapture('signup_form_filled_out_incorrect'); + + util.dismissAlert("Let me fix that..."); + + log.debug("------------------------------------"); + log.debug("Sign up: 3 character password."); + log.debug("------------------------------------"); + ob.signup().setValidEmailFormat(t.email); + ob.signup().setPassword('foo'); + ob.signup().selectCreateAccount(); + util.screenCapture('signup_form_password_short'); + util.dismissAlert("OK"); + // Cancel sign up. + assertTrue(ob.signup().selectCancel(),"Selected Cancel button."); + + // Bring sign up form back up. + assertTrue(ob.gotoSignup(),"At signup form."); + + // Set good password. + + log.debug("------------------------------------"); + log.debug("Sign up: Fix password and sign up."); + log.debug("------------------------------------"); + + ob.signup().setPassword(t.password); + ob.signup().selectCreateAccount(); + + util.screenCapture('created_account'); + + log.debug("------------------------------------"); + log.debug("Sign up: Get access code and enter it in form... Exit."); + log.debug("------------------------------------"); + var access_code = user.get_access_code('Email:' + t.email); + ob.signup().setAccessCode(access_code); + + // Confirm your account: exit + ob.signup().selectExit(); + + log.debug("------------------------------------"); + log.debug("Sign up: Enter 3-digit access code."); + log.debug("------------------------------------"); + ob.signup().selectCreateAccount(); + target.delay(4); // TODO: replace delay + ob.signup().setAccessCode('987'); + + ob.signup().selectContinue(); + util.screenCapture('access_code_too_short'); + util.dismissAlert("OK"); + + log.debug("------------------------------------"); + log.debug("Sign up: Send code again."); + log.debug("------------------------------------"); + ob.signup().selectSendCodeAgain(); + var access_code = user.get_access_code('Email:' + t.email); + + log.debug("------------------------------------"); + log.debug("Sign up: Enter correct access code."); + log.debug("------------------------------------"); + ob.signup().setAccessCode(access_code); + + ob.signup().selectContinue(); + target.delay(2); // for accurate screen shot + util.screenCapture('logged_in_dashboard'); + + + + + +}); diff --git a/clients/ios/testing/tests/testOnboardingSignUpExisting.js b/clients/ios/testing/tests/testOnboardingSignUpExisting.js new file mode 100644 index 0000000..0e5b92a --- /dev/null +++ b/clients/ios/testing/tests/testOnboardingSignUpExisting.js @@ -0,0 +1,37 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, login with new user (terminated) and + * an existing user + * @author: Greg Vandenberg + * + */ + +VFRunTest("testOnboardingSignUpExisting", runStateEnum.ONBOARDING, function(testname, util, log) { + + var ob = new VFOnboarding(util.getNav()); + var t = TEST_USERS[0]; + var user = new VFUser(); + user.register(t); + + log.debug("------------------------------------"); + log.debug("Start Signup process (existing user)."); + log.debug("------------------------------------"); + + ob.gotoSignup(); + util.screenCapture('signup_form2'); + + ob.signup().setFirstName(t.firstname); + ob.signup().setLastName(t.lastname); + ob.signup().setValidEmailFormat(t.email); + ob.signup().setPassword(t.password); + + util.screenCapture('signup_form_filled_out2'); + + ob.signup().selectCreateAccount(); + + util.screenCapture('created_account_existing'); + util.dismissAlert("OK"); + + ob.login().selectLogin(); +}); diff --git a/clients/ios/testing/tests/testProspectiveUser.js b/clients/ios/testing/tests/testProspectiveUser.js new file mode 100644 index 0000000..20722b3 --- /dev/null +++ b/clients/ios/testing/tests/testProspectiveUser.js @@ -0,0 +1,117 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, start a conversation, invite a + * prospective user, register user and see change in UI + * @author: Greg Vandenberg + * + */ + +VFRunTest("testProspectiveUser", runStateEnum.DASHBOARD, function(testname, util, log) { + var dash = util.gotoDashboard(); + var nav = util.getNav(); + var user = new VFUser(); + var t = TEST_USERS[1]; + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts."); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + log.debug("------------------------------------"); + log.debug("Create a new conversation."); + log.debug("------------------------------------"); + // goto conversation + var convo = dash.gotoConversations(); + + // compose new convo + convo.selectCompose(); + + log.debug("------------------------------------"); + log.debug("Invite a person to conversation."); + log.debug("------------------------------------"); + // add people to convo + convo.selectAddPeople(); + convo.setPerson(t.firstname + " "); + util.screenCapture('set_person_invite'); + convo.setPerson(t.lastname + "\n"); + util.screenCapture('set_person_invite_1'); + + // set title + convo.setTitle('Prospective User Test'); + + // Work around for issue #415 + convo.selectAddPeople(); + + // add photo to convo + convo.selectAddPhotos(); + + convo.selectNumImages(1); + + convo.selectAddPhotosNav(); + + convo.selectStart(); + + util.screenCapture('convo_started'); + + log.debug("------------------------------------"); + log.debug("Add comment to conversation"); + log.debug("------------------------------------"); + convo.addComment('test comment'); + util.screenCapture('comment_added'); + + convo.selectSend(); + convo.selectBack(); + util.gotoDashboard(); + util.screenCapture('dashboard_1_convo'); + + log.debug("------------------------------------"); + log.debug("Register User " + t.email); + log.debug("------------------------------------"); + user.register(t); + + log.debug("------------------------------------"); + log.debug("Background App to invoke query notifications"); + log.debug("------------------------------------"); + target.deactivateAppForDuration(2); + + dash.gotoConversations(); + + // compose new convo + convo.selectCompose(); + + log.debug("------------------------------------"); + log.debug("Show that client recognizes registered user."); + log.debug("------------------------------------"); + //add people to convo + convo.selectAddPeople(); + convo.setPerson(t.firstname + " "); + util.screenCapture('set_person_title_1'); + convo.setPerson(t.lastname + "\n"); + + // set title + convo.setTitle('Registered User Test'); + + util.screenCapture('set_person_title_2'); + + // Work around for issue #415 + convo.selectAddPeople(); + + // add photo to convo + convo.selectAddPhotos(); + + convo.selectNumImages(2); + + convo.selectAddPhotosNav(); + + convo.selectStart(); + + util.screenCapture('convo_started_1'); + +}); diff --git a/clients/ios/testing/tests/testRelatedConversations.js b/clients/ios/testing/tests/testRelatedConversations.js new file mode 100644 index 0000000..e1f0410 --- /dev/null +++ b/clients/ios/testing/tests/testRelatedConversations.js @@ -0,0 +1,100 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, start a conversation, access related conversation from library + * covers issue #461 + * @author: Greg Vandenberg + * + */ + +VFRunTest("testRelatedConversations", runStateEnum.DASHBOARD, function(testname, util, log) { + var dash = util.gotoDashboard(); + var nav = util.getNav(); + var user = new VFUser(); + var t = TEST_USERS[1]; + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + + // goto conversation + var convo = dash.gotoConversations(); + + log.debug("------------------------------------"); + log.debug("Create a new conversation."); + log.debug("------------------------------------"); + // compose new convo + convo.selectCompose(); + + log.debug("------------------------------------"); + log.debug("Add people to conversation."); + log.debug("------------------------------------"); + // add people to convo + convo.selectAddPeople(); + convo.setPerson(t.firstname + " " + t.lastname + "\n"); + + //set title + var convo_title = 'Test Conversation'; + convo.setTitle(convo_title); + + util.screenCapture('set_person_title'); + + //Work around for issue #415 + //convo.selectAddPeople(); + + log.debug("------------------------------------"); + log.debug("Add photo to conversation."); + log.debug("------------------------------------"); + // add photo to convo + convo.selectAddPhotos(); + convo.selectNumImages(3); + convo.selectAddPhotosNav(); + convo.selectStart(); + util.screenCapture('convo_started_1'); + + convo.selectBack(); + util.gotoDashboard(); + var library = dash.gotoLibrary(); + log.debug("------------------------------------"); + log.debug("Share 5 photos."); + log.debug("------------------------------------"); + library.selectActionButton(); + target.delay(2); // for accurate screen shot + util.screenCapture('library_action'); + + library.selectNumImages(5); + library.selectShareButton(); + library.selectNewConversation(); + //convo.selectAddPeople(); + convo.setPerson(t.firstname + " " + t.lastname + "\n"); + //set title + var convo_title = 'Another Conversation'; + convo.setTitle(convo_title); + convo.selectStart(); + util.screenCapture('convo_started_2'); + + convo.selectBack(); + util.gotoDashboard(); + + log.debug("------------------------------------"); + log.debug("Select Related Conversation."); + log.debug("------------------------------------"); + var library = dash.gotoLibrary(); + library.selectRelatedConversation(0); + util.screenCapture('select_related_conversation'); + library.selectConversation(); + util.screenCapture('show_conversation'); + convo.selectBack(); + util.gotoDashboard(); + //target.frontMostApp().mainWindow().scrollViews()[0].scrollViews()[1].buttons()["library related convos anchor"].tap(); + util.screenCapture('dashboard_1_convo'); + +}); diff --git a/clients/ios/testing/tests/testRemovedConversations.js b/clients/ios/testing/tests/testRemovedConversations.js new file mode 100644 index 0000000..a572a85 --- /dev/null +++ b/clients/ios/testing/tests/testRemovedConversations.js @@ -0,0 +1,101 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: From an unlinked state, + * + * User 1 starts a conversation, + * User 1 removes conversation, + * User 2 adds comment, + * Query Notifications, + * User 1 conversation is back in Inbox + * + * covers issue #461 + * @author: Greg Vandenberg + * + */ + +VFRunTest("testRemovedConversations", runStateEnum.DASHBOARD, function(testname, util, log) { + var dash = util.gotoDashboard(); + var nav = util.getNav(); + var user = new VFUser(); + var t = TEST_USERS[1]; + user.register(t); + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + + // goto conversation + var convo = dash.gotoConversations(); + + log.debug("------------------------------------"); + log.debug("Create a new conversation."); + log.debug("------------------------------------"); + // compose new convo + convo.selectCompose(); + + log.debug("------------------------------------"); + log.debug("Add people to conversation."); + log.debug("------------------------------------"); + // add people to convo + convo.selectAddPeople(); + convo.setPerson(t.firstname + " " + t.lastname + "\n"); + + //set title + var convo_title = 'Test Conversation'; + convo.setTitle(convo_title); + + util.screenCapture('set_person_title'); + + log.debug("------------------------------------"); + log.debug("Add photo to conversation."); + log.debug("------------------------------------"); + // add photo to convo + convo.selectAddPhotos(); + convo.selectNumImages(3); + convo.selectAddPhotosNav(); + convo.selectStart(); + util.screenCapture('convo_started_1'); + + convo.selectBack(); + target.delay(2); + convo.selectActionButton(); + target.delay(2); + log.debug("------------------------------------"); + log.debug("User 0 removes conversation."); + log.debug("------------------------------------"); + convo.selectCard(); + convo.selectRemoveButton(); + convo.selectConfirmRemoveButton(); + util.screenCapture('convo_removed'); + + log.debug("---------------------------------------------"); + log.debug("User 1 adds comment to conversation via api."); + log.debug("---------------------------------------------"); + convo.addServerComment('another test comment'); + + util.gotoDashboard(); + + log.debug("---------------------------------------------"); + log.debug("Invoke Query notifications."); + log.debug("---------------------------------------------"); + target.deactivateAppForDuration(1); + util.screenCapture('convo_notification'); + + log.debug("---------------------------------------------"); + log.debug("User 0 conversation is back in Inbox."); + log.debug("---------------------------------------------"); + dash.gotoConversations(); + util.screenCapture('convo_has_returned'); + + convo.selectCard(); + util.screenCapture('new_comment_added'); +}); diff --git a/clients/ios/testing/tests/testSettings.js b/clients/ios/testing/tests/testSettings.js new file mode 100644 index 0000000..0bf14e7 --- /dev/null +++ b/clients/ios/testing/tests/testSettings.js @@ -0,0 +1,84 @@ +/** + * + * @copyright 2013 Viewfinder Inc. All Rights Reserved. + * @description: Check all settings + * @author: Greg Vandenberg + * + */ +VFRunTest("testSettings", runStateEnum.DASHBOARD, function(testname, util, log) { + var dash = util.gotoDashboard(); + + log.debug("------------------------------------"); + log.debug("Show initial Setting screen."); + log.debug("------------------------------------"); + var settings = dash.gotoSettings(); + util.screenCapture('settings_screen'); + + settings.selectStorage('local'); + + log.debug("------------------------------------"); + log.debug("Select lowest option on picker."); + log.debug("------------------------------------"); + settings.selectLowestPickerOption(); + util.screenCapture('Lowest_Picker_Option'); + + settings.selectStorage('local'); + + log.debug("------------------------------------"); + log.debug("Turn off Cloud Storage."); + log.debug("------------------------------------"); + settings.selectStorage('cloud'); + + //settings.selectViewfinderPlus(); + //util.screenCapture('viewfinder_plus_option'); + //target.frontMostApp().windows()[0].tableViews()["Empty list"].cells()["Cloud Storage, 1 GB"].tap(); + //target.frontMostApp().windows()[0].tableViews()["Empty list"].cells()["5 GB, Viewfinder Plus, $1.99 / month"].tap(); + + settings.selectCloudStorage(1); + util.screenCapture('cloud_storage_on'); + + settings.selectCloudStorage(0); + util.screenCapture('cloud_storage_off'); + + util.selectBackNav(); + + // TODO: add buy flow + //target.frontMostApp().windows()[0].navigationBar().buttons()["Buy"].tap(); + //util.waitUntilVisible(SETTINGS_PAGE_FAQ, 5); + + log.debug("------------------------------------"); + log.debug("Goto FAQ."); + log.debug("------------------------------------"); + settings.selectSubPage(SETTINGS_PAGE_FAQ); + target.delay(3); // TODO: poll instead + util.screenCapture('FAQ'); + settings.selectBackNav(); + + log.debug("------------------------------------"); + log.debug("Goto Feedback."); + log.debug("------------------------------------"); + settings.selectSubPage(SETTINGS_PAGE_FEEDBACK); + target.delay(3); // TODO: poll instead + util.screenCapture('feedback_email'); + util.selectBackNav(); + settings.selectDeleteDraft(); + + log.debug("------------------------------------"); + log.debug("Goto Terms of Service."); + log.debug("------------------------------------"); + settings.selectSubPage(SETTINGS_PAGE_TOS); + target.delay(3); // TODO: poll instead + util.screenCapture('TOS'); + settings.selectBackNav(); + + log.debug("------------------------------------"); + log.debug("Goto Privacy Policy."); + log.debug("------------------------------------"); + settings.selectSubPage(SETTINGS_PAGE_PRIVACY); + util.screenCapture('privacy_policy'); + settings.selectBackNav(); + + + + +}); diff --git a/clients/ios/testing/tests/testSingleImageLandscapeView.js b/clients/ios/testing/tests/testSingleImageLandscapeView.js new file mode 100644 index 0000000..d6da23d --- /dev/null +++ b/clients/ios/testing/tests/testSingleImageLandscapeView.js @@ -0,0 +1,92 @@ +/** + * + * Copyright 2013 Viewfinder Inc. All Rights Reserved. + * Description: Single image view in the Library. + * Author: Greg Vandenberg + * + */ + +VFRunTest("testSingleImageLandscapeView", runStateEnum.DASHBOARD, function(testname, util, log) { + + var dash = new VFDashboard(); + var user = new VFUser(); + var t = TEST_USERS[1]; + var nav = util.getNav(); + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + log.debug("------------------------------------"); + log.debug("Go to Library."); + log.debug("------------------------------------"); + var library = dash.gotoLibrary(); + target.delay(2); // for accurate screen shot + util.screenCapture('personal_library'); + + log.debug("------------------------------------"); + log.debug("Select 1 image."); + log.debug("------------------------------------"); + library.selectNumImages(1); + + log.debug("------------------------------------"); + log.debug("Landscape single image view."); + log.debug("------------------------------------"); + target.setDeviceOrientation(UIA_DEVICE_ORIENTATION_LANDSCAPERIGHT); + util.screenCapture('single_image_landscape_view'); + + + // swipe right x images + while(1) { + nav.swipeRightToLeft(); + var sText = target.main().scrollViews()[0].staticTexts()[" 9:17 PM - Sun, December 23, 2012"]; + var rect = null; + if (util.type(sText) != 'UIAElementNil') { + if (sText.isVisible()) { + UIALogger.logDebug("Found image."); + break; + } + } + } + + target.delay(2); + library.selectRemoveButton(); + library.selectConfirmRemove(); + + target.delay(2); + //target.logElementTree(); + + // start conversation from portrait single image view + library.selectShareButton(); + var convo = new VFConversation(nav); + //add people to convo + convo.selectAddPeople(); + convo.setPerson(t.email + "\n"); + + //set title + convo.setTitle('Test Conversation'); + + util.screenCapture('set_person_title'); + + //Work around for issue #415 + convo.selectAddPeople(); + convo.selectStart(); + + util.screenCapture('convo_started'); + + convo.selectBack(); + + library.selectBackButton('main_window'); + + util.gotoDashboard(); + util.screenCapture('dashboard_1_convo'); + + +}); diff --git a/clients/ios/testing/tests/testSingleImagePortraitView.js b/clients/ios/testing/tests/testSingleImagePortraitView.js new file mode 100644 index 0000000..403f7a2 --- /dev/null +++ b/clients/ios/testing/tests/testSingleImagePortraitView.js @@ -0,0 +1,99 @@ +/** + * + * Copyright 2013 Viewfinder Inc. All Rights Reserved. + * Description: Single image view in the Library. + * Author: Greg Vandenberg + * + */ + +VFRunTest("testSingleImagePortraitView", runStateEnum.DASHBOARD, function(testname, util, log) { + + var dash = new VFDashboard(); + var user = new VFUser(); + var t = TEST_USERS[1]; + var nav = util.getNav(); + + log.debug("------------------------------------"); + log.debug("Import phonebook contacts"); + log.debug("------------------------------------"); + var contacts = dash.gotoContacts() + contacts.importPhoneContacts(); + util.screenCapture("at_import_contacts"); + + contacts.selectBackNav(); + util.dismissAlert("View All"); + contacts.gotoDashboard(); + + log.debug("------------------------------------"); + log.debug("Go to Library."); + log.debug("------------------------------------"); + var library = dash.gotoLibrary(); + target.delay(2); // for accurate screen shot + util.screenCapture('personal_library'); + + log.debug("------------------------------------"); + log.debug("Select 1 image."); + log.debug("------------------------------------"); + library.selectNumImages(1); + + + log.debug("------------------------------------"); + log.debug("Portrait single image view."); + log.debug("------------------------------------"); + target.delay(2); + target.setDeviceOrientation(UIA_DEVICE_ORIENTATION_PORTRAIT); + util.screenCapture('single_image_portrait_view'); + + // swipe right x images + while(1) { + nav.swipeRightToLeft(); + var sText = target.main().scrollViews()[0].staticTexts()[" 9:17 PM - Sun, December 23, 2012"]; + var rect = null; + if (util.type(sText) != 'UIAElementNil') { + if (sText.isVisible()) { + UIALogger.logDebug("Found image."); + break; + } + } + } + + log.debug("------------------------------------"); + log.debug("Remove Tesla Model S image."); + log.debug("------------------------------------"); + target.delay(2); + library.selectRemoveButton(); + library.selectConfirmRemove(); + + target.delay(2); + //target.logElementTree(); + + log.debug("------------------------------------"); + log.debug("Share image from single image view."); + log.debug("------------------------------------"); + // start conversation from portrait single image view + library.selectShareButton(); + var convo = new VFConversation(nav); + //add people to convo + convo.selectAddPeople(); + convo.setPerson(t.email + "\n"); + + //set title + convo.setTitle('Test Conversation'); + + util.screenCapture('set_person_title'); + + //Work around for issue #415 + convo.selectAddPeople(); + convo.selectStart(); + target.delay(1); + util.screenCapture('convo_started'); + + convo.selectBack(); + + library.selectBackButton('main_window'); + + util.gotoDashboard(); + util.screenCapture('dashboard_1_convo'); + + +}); diff --git a/clients/ios/testplan.txt b/clients/ios/testplan.txt new file mode 100644 index 0000000..5db5329 --- /dev/null +++ b/clients/ios/testplan.txt @@ -0,0 +1,142 @@ +Basic Walk Through +- iOS 5.0 Simulator +- iOS 5.1 Simulator +- iOS 6.0 Simulator +- iOS 6.1 Simulator + +Camera +- Flash on +- Flash off +- Flash auto +- Back camera, portrait +- Back camera, portrait upside-down +- Back camera, landscape left +- Back camera, landscape right +- Front camera, portrait +- Front camera, portrait upside-down +- Front camera, landscape left +- Front camera, landscape right + +Contacts +- Import from gmail +- Import from facebook +- All vs Viewfinder +- Autocomplete +- Set nickname +- Clear nickname +- Conversations List + +Conversation View +- Add comment +- Add photo +- Export photo +- Unshare photo + +Event View +- Share new +- Share existing +- Export photo +- Remove photo + +Import Photos +- Import Photos from Camera Roll +- Import Photos from Synced Folder + - Verify duplicates are not created +- Import Photos from iPhoto Event + +Photo Selection +- Tapping photo toggles selection +- Tapping title area toggles selection of group +- Swipe left selects none +- Swipe right selects all +- Tapping reply-to-photo toggles selection of both thumbnail and original photo +- Tapping cover photo toggles selection of both cover photo and the thumbnail + +Settings +- Edit display name + - Verify done and dismiss buttons work + - Pan to dismiss keyboard, verify done and dismiss buttons disappear +- Local storage + - Pan to dismiss + + +Sidebar +- Title bar + - User icon + user name if signed in + - Otherwise, "Sign Up and Start Sharing..." +- Library +- Conversations +- Contacts +- Settings +- Restart Tutorial +- Send Feedback + +Sign-up/Login + +Single Photo View +- Share new +- Share existing +- Export photo +- Remove photo +- Unshare photo + +Summary (Library/Inbox) View +- Viewfinder button +- Sidebar button +- Title scrolls to top, then toggles between views +- Swipe to toggle between views + - Swipe while scrolling +- Activate jump scroll via long press on right margin +- Activate dial via long press on left margin + - Toggle between elastic and non-elastic dial + + +Walk-through Script +------------------- + +1. Remove any existing Viewfinder app. +2. Go to Settings > General > Reset > Reset Location & Privacy. +3. Install Viewfinder. +4. Verify "Your Photos" page. +5. Verify "Access Your Photos" alert. + a. Click "Don't Allow". + - Verify "Well this is boring..." page. + - Settings > Privacy > Photos > Viewfinder + - Re-enter app + - Verify "Library" appears. + b. Click "OK". + - Verify "Library" appears. +6. Tap title to toggle between library and conversations. + a. Verify "Start a Conversation" page is displayed for conversations list. + b. Verify "Sign up and start sharing" button works. +7. Tap sidebar button. +8. Verify sidebar can be closed via dragging. +9. Verify "Sign Up and Start Sharing..." at top of sidebar. + a. Tap "Sign Up and Start Sharing". + b. Verify "Create Account/I Have An Account" page appears. +10. Tap "Send Feedback" + a. Verify feedback email template appears. +11. Tap "Start Tutorial" + a. Verify tutorial is started. +12. Tap "Settings" + a. Verify Settings page contains "Local Storage" and "Legalese" sections. + b. Verify "Local Storage" value can be changed. + c. Verify "Terms of Service" can be viewed. + d. Verify "Privacy Policy" can be viewed. +13. Tap "Contacts" + a. Verify "Start a Conversation" appears. + b. Verify "Sign up and start sharing" button works. +14. Navigate to Camera + a. Verify "Location Services Authorization" alert is displayed + b. Take a photo + - Flash on + - Flash off + - Flash auto + - Back camera, portrait + - Back camera, portrait upside-down + - Back camera, landscape left + - Back camera, landscape right + - Front camera, portrait + - Front camera, portrait upside-down + - Front camera, landscape left + - Front camera, landscape right diff --git a/clients/ios/testplan/client-setup b/clients/ios/testplan/client-setup new file mode 100644 index 0000000..b435842 --- /dev/null +++ b/clients/ios/testplan/client-setup @@ -0,0 +1,13 @@ +Compile and install iOS client: + +On client build machine: + +hg fetch to get latest source code +Comment out "PRODUCTION" in ~/.viewfinder.DeveloperDefines.h + + +On each of two test devices: + +Remove any existing Viewfinder app +Go to Settings > General > Reset > Reset Location & Privacy +Compile and run client to phone diff --git a/clients/ios/testplan/not-registered/0001 b/clients/ios/testplan/not-registered/0001 new file mode 100644 index 0000000..124ccbd --- /dev/null +++ b/clients/ios/testplan/not-registered/0001 @@ -0,0 +1,9 @@ +Verify expected contents of library <1> + +Verify conversations page has signup link <2> + +Verify sidebar has signup link <3> + +Verify contacts has signup link + +Verify empty settings <5> \ No newline at end of file diff --git a/clients/ios/testplan/not-registered/photo 1.PNG b/clients/ios/testplan/not-registered/photo 1.PNG new file mode 100644 index 0000000..615e56d Binary files /dev/null and b/clients/ios/testplan/not-registered/photo 1.PNG differ diff --git a/clients/ios/testplan/not-registered/photo 2.PNG b/clients/ios/testplan/not-registered/photo 2.PNG new file mode 100644 index 0000000..d8af5f5 Binary files /dev/null and b/clients/ios/testplan/not-registered/photo 2.PNG differ diff --git a/clients/ios/testplan/not-registered/photo 3.PNG b/clients/ios/testplan/not-registered/photo 3.PNG new file mode 100644 index 0000000..af5fbd8 Binary files /dev/null and b/clients/ios/testplan/not-registered/photo 3.PNG differ diff --git a/clients/ios/testplan/not-registered/photo 5.PNG b/clients/ios/testplan/not-registered/photo 5.PNG new file mode 100644 index 0000000..d90e1c3 Binary files /dev/null and b/clients/ios/testplan/not-registered/photo 5.PNG differ diff --git a/clients/ios/testplan/server-setup b/clients/ios/testplan/server-setup new file mode 100644 index 0000000..af94d00 --- /dev/null +++ b/clients/ios/testplan/server-setup @@ -0,0 +1,8 @@ +Start local viewfinder server: + +scripts/local-viewfinder-prep --localdb_reset +scripts/local-viewfinder + +You can access the administrative pages at: + +https://www.goviewfinder.com:8443/admin diff --git a/clients/ios/testplan/startup/0001 b/clients/ios/testplan/startup/0001 new file mode 100644 index 0000000..8c607e1 --- /dev/null +++ b/clients/ios/testplan/startup/0001 @@ -0,0 +1,3 @@ +Verify "Your Photos" page appears <1> + +Click "Understood" \ No newline at end of file diff --git a/clients/ios/testplan/startup/0002 b/clients/ios/testplan/startup/0002 new file mode 100644 index 0000000..8a5c4da --- /dev/null +++ b/clients/ios/testplan/startup/0002 @@ -0,0 +1,5 @@ +Verify "Viewfinder" Would Like to Access Your Photos <2> + +Click "Don't Allow" + +Verify "Well this is boring..." <3> \ No newline at end of file diff --git a/clients/ios/testplan/startup/0003 b/clients/ios/testplan/startup/0003 new file mode 100644 index 0000000..4d013d4 --- /dev/null +++ b/clients/ios/testplan/startup/0003 @@ -0,0 +1,5 @@ +Switch to Settings app and go "Privacy > Photos" <4> + +Enable "Viewfinder" + +Switch back to client, verify it loads photos. \ No newline at end of file diff --git a/clients/ios/testplan/startup/photo 1.PNG b/clients/ios/testplan/startup/photo 1.PNG new file mode 100644 index 0000000..5caa26f Binary files /dev/null and b/clients/ios/testplan/startup/photo 1.PNG differ diff --git a/clients/ios/testplan/startup/photo 2.PNG b/clients/ios/testplan/startup/photo 2.PNG new file mode 100644 index 0000000..3e3bc01 Binary files /dev/null and b/clients/ios/testplan/startup/photo 2.PNG differ diff --git a/clients/ios/testplan/startup/photo 3.PNG b/clients/ios/testplan/startup/photo 3.PNG new file mode 100644 index 0000000..ed42378 Binary files /dev/null and b/clients/ios/testplan/startup/photo 3.PNG differ diff --git a/clients/ios/testplan/startup/photo 4.PNG b/clients/ios/testplan/startup/photo 4.PNG new file mode 100644 index 0000000..82e3622 Binary files /dev/null and b/clients/ios/testplan/startup/photo 4.PNG differ diff --git a/clients/ios/third_party b/clients/ios/third_party new file mode 120000 index 0000000..b19b9e9 --- /dev/null +++ b/clients/ios/third_party @@ -0,0 +1 @@ +../../third_party/ios/ \ No newline at end of file diff --git a/clients/ios/third_party_shared b/clients/ios/third_party_shared new file mode 120000 index 0000000..948df75 --- /dev/null +++ b/clients/ios/third_party_shared @@ -0,0 +1 @@ +../../third_party/shared/ \ No newline at end of file diff --git a/clients/shared/ActivityMetadata.proto b/clients/shared/ActivityMetadata.proto new file mode 100644 index 0000000..e085cdb --- /dev/null +++ b/clients/shared/ActivityMetadata.proto @@ -0,0 +1,106 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +import "ContentIds.proto"; +import "ContactMetadata.proto"; +import "QueueMetadata.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ActivityMetadataPB"; + +message ActivityMetadata { + message Episode { + optional EpisodeId episode_id = 1; + repeated PhotoId photo_ids = 2; + } + + message AddFollowers { + // On receipt from the server, only user_id will be set in each + // contact metadata. However, on a locally-generated activity, + // the full contact metadata is set, which allows contacts to be + // named without necessarily having a user-id yet for the intended + // follower. + repeated ContactMetadata contacts = 1; + } + + message MergeAccounts { + optional int64 source_user_id = 1; + optional int64 target_user_id = 2; + } + + message PostComment { + optional CommentId comment_id = 1; + } + + message RemoveFollowers { + repeated int64 user_ids = 1; + } + + message SavePhotos { + repeated Episode episodes = 1; + // For viewpoint autosave. + optional ViewpointId viewpoint_id = 2; + } + + message ShareNew { + repeated Episode episodes = 1; + repeated ContactMetadata contacts = 2; + } + + message ShareExisting { + repeated Episode episodes = 1; + } + + message Unshare { + repeated Episode episodes = 1; + } + + message UpdateEpisode { + optional EpisodeId episode_id = 1; + } + + message UpdateViewpoint { + optional ViewpointId viewpoint_id = 1; + } + + message UploadEpisode { + optional EpisodeId episode_id = 1; + repeated PhotoId photo_ids = 2; + } + + // NOTE: if any new activity type is added, make sure to also clear + // it in ActivityTable_Activity::MergeFrom. + + optional ActivityId activity_id = 1; + optional ViewpointId viewpoint_id = 2; + optional int64 user_id = 3; + optional double timestamp = 4; + optional int64 update_seq = 5; + optional QueueMetadata queue = 6; + + // NOTE: if any new activities are added here, be sure to update + // ActivityTable_Activity::MergeFrom. + optional AddFollowers add_followers = 7; + optional MergeAccounts merge_accounts = 16; + optional PostComment post_comment = 8; + optional RemoveFollowers remove_followers = 17; + optional SavePhotos save_photos = 15; + optional ShareNew share_new = 9; + optional ShareExisting share_existing = 10; + optional Unshare unshare = 11; + optional UpdateEpisode update_episode = 12; + optional UpdateViewpoint update_viewpoint = 13; + optional UploadEpisode upload_episode = 14; + + optional bool label_error = 20; + + // Provisional activities are not uploaded to the server until they become + // permanent. + optional bool provisional = 32; + // A boolean indicating whether the activity needs to be uploaded to the + // server or not. + optional bool upload_activity = 30; + + // Timestamp at which this activity was viewed on this device. + optional double viewed_timestamp = 31; +} diff --git a/clients/shared/ActivityTable.cc b/clients/shared/ActivityTable.cc new file mode 100644 index 0000000..96dccb4 --- /dev/null +++ b/clients/shared/ActivityTable.cc @@ -0,0 +1,1070 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import "ActivityTable.h" +#import "AppState.h" +#import "CommentTable.h" +#import "ContactManager.h" +#import "DayTable.h" +#import "EpisodeTable.h" +#import "NetworkQueue.h" +#import "PlacemarkHistogram.h" +#import "StringUtils.h" +#import "WallTime.h" + +namespace { + +const int kActivityFSCKVersion = 6; + +const int kMaxContentLength = 30; + +const string kActivityTimestampKeyPrefix = DBFormat::activity_timestamp_key(""); +const string kCommentActivityKeyPrefix = DBFormat::comment_activity_key(""); +const string kEpisodeActivityKeyPrefix = DBFormat::episode_activity_key(""); +const string kQuarantinedActivityKeyPrefix = DBFormat::quarantined_activity_key(""); +const string kViewpointActivityKeyPrefix = DBFormat::viewpoint_activity_key(""); + +const DBRegisterKeyIntrospect kActivityKeyIntrospect( + DBFormat::activity_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kActivityServerKeyIntrospect( + DBFormat::activity_server_key(), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kActivityTimestampKeyIntrospect( + kActivityTimestampKeyPrefix, [](Slice key) { + WallTime timestamp; + int64_t activity_id; + if (!DecodeActivityTimestampKey(key, ×tamp, &activity_id)) { + return string(); + } + return string( + Format("%s/%d", DBIntrospect::timestamp(timestamp), activity_id)); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointActivityKeyIntrospect( + kViewpointActivityKeyPrefix, [](Slice key) { + WallTime timestamp; + int64_t activity_id; + int64_t viewpoint_id; + if (!DecodeViewpointActivityKey(key, &viewpoint_id, ×tamp, &activity_id)) { + return string(); + } + return string( + Format("%d/%s/%d", viewpoint_id, + DBIntrospect::timestamp(timestamp), activity_id)); + }, NULL); + +const DBRegisterKeyIntrospect kCommentActivityKeyIntrospect( + DBFormat::comment_activity_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kEpisodeActivityKeyIntrospect( + DBFormat::episode_activity_key(""), [](Slice key) { + string server_episode_id; + int64_t activity_id; + if (!DecodeEpisodeActivityKey(key, &server_episode_id, &activity_id)) { + return string(); + } + return string( + Format("[%s]/%d", ServerIdFormat(server_episode_id), activity_id)); + }, NULL); + +const DBRegisterKeyIntrospect kQuarantinedActivityKeyIntrospect( + kQuarantinedActivityKeyPrefix, [](Slice key) { + int64_t activity_id; + if (!DecodeQuarantinedActivityKey(key, &activity_id)) { + return string(); + } + return string(Format("%d", activity_id)); + }, NULL); + +// Iterates over activities through a time range, ordered from most +// recent to least recent. +// +// Since timestamps are sorted in reverse order, start_time should be +// greater than or equal to end_time. +class TimestampActivityIterator : public ActivityTable::ActivityIterator { + public: + TimestampActivityIterator(ActivityTable* table, WallTime start_time, + bool reverse, const DBHandle& db) + : ActivityTable::ActivityIterator(table, reverse, db) { + Seek(start_time); + } + + void Seek(WallTime seek_time) { + ContentIterator::Seek(EncodeActivityTimestampKey( + seek_time, reverse_ ? std::numeric_limits::max() : 0)); + } + + protected: + virtual bool IteratorDone(const Slice& key) { + return !key.starts_with(kActivityTimestampKeyPrefix); + } + + virtual bool UpdateStateHook(const Slice& key) { + return DecodeActivityTimestampKey(key, &cur_timestamp_, &cur_activity_id_); + } +}; + +// Iterates over activites in a viewpoint in order of increasing +// activity id. +class ViewpointActivityIterator : public ActivityTable::ActivityIterator { + public: + ViewpointActivityIterator(ActivityTable* table, int64_t viewpoint_id, + WallTime start_time, bool reverse, const DBHandle& db) + : ActivityTable::ActivityIterator(table, reverse, db), + viewpoint_id_(viewpoint_id) { + Seek(start_time); + } + + void Seek(WallTime seek_time) { + ContentIterator::Seek(EncodeViewpointActivityKey( + viewpoint_id_, seek_time, + reverse_ ? std::numeric_limits::max() : 0)); + } + + protected: + virtual bool IteratorDone(const Slice& key) { + int64_t viewpoint_id; + if (!DecodeViewpointActivityKey(key, &viewpoint_id, &cur_timestamp_, &cur_activity_id_)) { + return true; + } + return viewpoint_id != viewpoint_id_; + } + + virtual bool UpdateStateHook(const Slice& key) { + int64_t viewpoint_id; + return DecodeViewpointActivityKey(key, &viewpoint_id, &cur_timestamp_, &cur_activity_id_); + } + + private: + const int64_t viewpoint_id_; +}; + +} // namespace + + +string EncodeActivityTimestampKey( + WallTime timestamp, int64_t activity_id) { + string s; + OrderedCodeEncodeVarint32(&s, timestamp); + OrderedCodeEncodeVarint64(&s, activity_id); + return DBFormat::activity_timestamp_key(s); +} + +string EncodeCommentActivityKey(const string& comment_server_id) { + return DBFormat::comment_activity_key(comment_server_id); +} + +string EncodeEpisodeActivityKey( + const string& episode_server_id, int64_t activity_id) { + string s = episode_server_id; + s += '/'; + OrderedCodeEncodeVarint64(&s, activity_id); + return DBFormat::episode_activity_key(s); +} + +string EncodeEpisodeActivityKeyPrefix(const string& episode_server_id) { + string s = episode_server_id; + s += '/'; + return DBFormat::episode_activity_key(s); +} + +string EncodeQuarantinedActivityKey(int64_t activity_id) { + string s; + OrderedCodeEncodeVarint64(&s, activity_id); + return DBFormat::quarantined_activity_key(s); +} + +string EncodeViewpointActivityKey( + int64_t viewpoint_id, WallTime timestamp, int64_t activity_id) { + string s; + OrderedCodeEncodeVarint64(&s, viewpoint_id); + OrderedCodeEncodeVarint32(&s, timestamp); + OrderedCodeEncodeVarint64(&s, activity_id); + return DBFormat::viewpoint_activity_key(s); +} + +bool DecodeActivityTimestampKey( + Slice key, WallTime* timestamp, int64_t* activity_id) { + if (!key.starts_with(kActivityTimestampKeyPrefix)) { + return false; + } + key.remove_prefix(kActivityTimestampKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32(&key); + *activity_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeCommentActivityKey(Slice key, string* server_comment_id) { + if (!key.starts_with(kCommentActivityKeyPrefix)) { + return false; + } + key.remove_prefix(kCommentActivityKeyPrefix.size()); + *server_comment_id = key.ToString(); + return true; +} + +bool DecodeEpisodeActivityKey( + Slice key, string* server_episode_id, int64_t* activity_id) { + if (!key.starts_with(kEpisodeActivityKeyPrefix)) { + return false; + } + key.remove_prefix(kEpisodeActivityKeyPrefix.size()); + int slash_pos = key.find('/'); + if (slash_pos == -1) { + return false; + } + *server_episode_id = key.substr(0, slash_pos).ToString(); + key.remove_prefix(slash_pos + 1); + *activity_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeQuarantinedActivityKey(Slice key, int64_t* activity_id) { + if (!key.starts_with(kQuarantinedActivityKeyPrefix)) { + return false; + } + key.remove_prefix(kQuarantinedActivityKeyPrefix.size()); + *activity_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeViewpointActivityKey( + Slice key, int64_t* viewpoint_id, WallTime* timestamp, int64_t* activity_id) { + if (!key.starts_with(kViewpointActivityKeyPrefix)) { + return false; + } else { + key.remove_prefix(kViewpointActivityKeyPrefix.size()); + } + *viewpoint_id = OrderedCodeDecodeVarint64(&key); + *timestamp = OrderedCodeDecodeVarint32(&key); + *activity_id = OrderedCodeDecodeVarint64(&key); + return true; +} + + +//// +// Activity + +ActivityTable_Activity::ActivityTable_Activity( + AppState* state, const DBHandle& db, int64_t id) + : state_(state), + db_(db), + disk_timestamp_(0) { + mutable_activity_id()->set_local_id(id); +} + +void ActivityTable_Activity::MergeFrom(const ActivityMetadata& m) { + // Clear out existing activity so merge doesn't accrete duplicate content. + // NOTE: add activities here as they're added to the protobuf. + clear_add_followers(); + clear_merge_accounts(); + clear_post_comment(); + clear_save_photos(); + clear_share_new(); + clear_share_existing(); + clear_unshare(); + clear_update_episode(); + clear_update_viewpoint(); + clear_upload_episode(); + + // Some assertions that immutable properties of the activity don't change. + if (has_timestamp() && m.has_timestamp()) { + DCHECK_LT(fabs(timestamp() - m.timestamp()), 1e-6); + } + if (viewpoint_id().has_server_id() && m.viewpoint_id().has_server_id()) { + DCHECK_EQ(viewpoint_id().server_id(), m.viewpoint_id().server_id()); + } + if (has_user_id() && m.has_user_id()) { + DCHECK_EQ(user_id(), m.user_id()); + } + + ActivityMetadata::MergeFrom(m); +} + +void ActivityTable_Activity::MergeFrom(const ::google::protobuf::Message&) { + DIE("MergeFrom(Message&) should not be used"); +} + +bool ActivityTable_Activity::FilterShare( + const PhotoSelectionSet& selection, const DBHandle& updates) { + DCHECK(provisional()); + if (!provisional()) { + return false; + } + if (!has_share_new() && !has_share_existing()) { + return false; + } + + // Loop over the photos for the share and remove any that are no longer part + // of the selection. We maintain the existing order by building up a new + // ShareExisting activity from the old one. + ShareEpisodes old_share_episodes; + ShareEpisodes new_share_episodes; + if (has_share_new()) { + old_share_episodes.Swap(mutable_share_new()->mutable_episodes()); + } else if (has_share_existing()) { + old_share_episodes.Swap(mutable_share_existing()->mutable_episodes()); + } + + for (int i = 0; i < old_share_episodes.size(); ++i) { + const ActivityMetadata::Episode& old_episode = old_share_episodes.Get(i); + ActivityMetadata::Episode* new_episode = NULL; + const EpisodeId& episode_id = old_episode.episode_id(); + EpisodeHandle eh; + + for (int j = 0; j < old_episode.photo_ids_size(); ++j) { + const PhotoId& photo_id = old_episode.photo_ids(j); + const PhotoSelection key(photo_id.local_id(), episode_id.local_id()); + if (!ContainsKey(selection, key)) { + // The photo is not part of the selection and should be removed from + // the episode. It is removed from the activity by simply not copying + // it to the new share activity. + if (!eh.get()) { + eh = state_->episode_table()->LoadEpisode(episode_id, updates); + eh->Lock(); + } + if (eh.get()) { + eh->RemovePhoto(photo_id.local_id()); + } + continue; + } + + if (!new_episode) { + new_episode = new_share_episodes.Add(); + new_episode->mutable_episode_id()->CopyFrom(episode_id); + } + new_episode->add_photo_ids()->CopyFrom(photo_id); + } + + if (eh.get()) { + // Delete the episode/activity index key. + updates->Delete(EncodeEpisodeActivityKey(episode_id.server_id(), local_id())); + eh->SaveAndUnlock(updates); + } + } + + if (has_share_new()) { + if (new_share_episodes.size() == 0) { + if (share_new().contacts_size() == 0) { + clear_share_new(); + } + return false; + } + mutable_share_new()->mutable_episodes()->Swap(&new_share_episodes); + } else if (has_share_existing()) { + if (new_share_episodes.size() == 0) { + clear_share_existing(); + return false; + } + mutable_share_existing()->mutable_episodes()->Swap(&new_share_episodes); + } + return true; +} + +string ActivityTable_Activity::FormatName(bool shorten) { + if (shorten) { + return state_->contact_manager()->FirstName(user_id()); + } else { + return state_->contact_manager()->FullName(user_id()); + } +} + +string ActivityTable_Activity::FormatTimestamp(bool shorten) { + // TODO(spencer): need a shortened version of the relative time method. + return FormatRelativeTime(timestamp(), state_->WallTime_Now()); +} + +string ActivityTable_Activity::FormatShareContent( + const ViewpointSummaryMetadata::ActivityRow* activity_row, bool shorten) { + int count = 0; + vector episodes; + + // Use the activity row photos if possible; otherwise, use activity episodes. + if (activity_row) { + std::unordered_set unique_episode_ids; + for (int i = 0; i < activity_row->photos_size(); ++i) { + ++count; + const int64_t episode_id = activity_row->photos(i).episode_id(); + if (ContainsKey(unique_episode_ids, episode_id)) { + continue; + } + unique_episode_ids.insert(episode_id); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(episode_id, db_); + if (eh.get()) { + episodes.push_back(eh); + } + } + } else { + const ShareEpisodes* share_episodes = GetShareEpisodes(); + for (int i = 0; i < share_episodes->size(); ++i) { + count += share_episodes->Get(i).photo_ids_size(); + EpisodeHandle eh = + state_->episode_table()->LoadEpisode(share_episodes->Get(i).episode_id(), db_); + if (eh.get()) { + episodes.push_back(eh); + } + } + } + + string long_loc_str; + string first_loc_str; + std::unordered_set unique_loc_strs; + for (int i = 0; i < episodes.size(); ++i) { + Location location; + Placemark placemark; + EpisodeHandle eh = episodes[i]; + if (eh->GetLocation(&location, &placemark) && + (placemark.has_sublocality() || + placemark.has_locality() || + placemark.has_state() || + placemark.has_country())) { + if (long_loc_str.empty()) { + state_->placemark_histogram()->FormatLocation( + location, placemark, shorten, &long_loc_str); + } + string short_loc_str; + state_->placemark_histogram()->FormatLocation( + location, placemark, true, &short_loc_str); + if (!short_loc_str.empty()) { + if (first_loc_str.empty()) { + first_loc_str = short_loc_str; + } + unique_loc_strs.insert(short_loc_str); + } + } + } + + if (unique_loc_strs.empty() && count > 0) { + return Format("%d photo%s without location%s", + count, Pluralize(count), Pluralize(count)); + } else if (unique_loc_strs.size() == 1) { + DCHECK(!long_loc_str.empty()); + return ToUppercase(long_loc_str); + } else { + return Format("%s and %d other location%s", ToUppercase(first_loc_str), + unique_loc_strs.size() - 1, Pluralize(unique_loc_strs.size() - 1)); + } +} + +string ActivityTable_Activity::FormatContent( + const ViewpointSummaryMetadata::ActivityRow* activity_row, bool shorten) { + if (has_share_new() || has_share_existing()) { + return FormatShareContent(activity_row, shorten); + } else if (has_post_comment()) { + CommentHandle ch = state_->comment_table()->LoadComment( + post_comment().comment_id(), db_); + if (ch.get()) { + if (shorten) { + const string normalized = NormalizeWhitespace(ch->message()); + if (normalized.length() <= kMaxContentLength) { + // Optimization: the byte length of a string is never more than its character length, so + // if we have less than kMaxContentLength bytes we can skip the character counting. + return normalized.empty() ? " " : normalized; + } else { + const string truncated = TruncateUTF8(ch->message(), kMaxContentLength); + if (truncated.length() == ch->message().length()) { + // If the message contains multi-byte characters it may not have actually needed truncation. + return truncated; + } else { + // If it did, remove any now-trailing whitespace (using the more efficient Trim since + // the previous NormalizeWhitespace got rid of any non-ascii whitespace) and add an + // ellipsis. + return Trim(truncated) + "…"; + } + } + } + const string trimmed = Trim(ch->message()); + return trimmed.empty() ? " " : trimmed; + } else { + return " "; + } + } else if (has_save_photos()) { + return Format("%saved photos", shorten ? "S" : "s"); + } else if (has_update_episode()) { + // TODO(spencer): need something better here. + return Format("%spdated episode", shorten ? "U" : "u"); + } else if (has_update_viewpoint()) { + // TODO(spencer): need something better here. + return Format("%spdated conversation", shorten ? "U" : "u"); + } else if (has_add_followers()) { + const bool more_than_one_follower = add_followers().contacts_size() > 1; + vector followers; + for (int i = 0; i < add_followers().contacts_size(); ++i) { + if (add_followers().contacts(i).has_user_id()) { + const int64_t user_id = add_followers().contacts(i).user_id(); + if (more_than_one_follower) { + followers.push_back(state_->contact_manager()->FirstName(user_id)); + } else { + followers.push_back(state_->contact_manager()->FullName(user_id)); + } + } else { + followers.push_back(ContactManager::FormatName( + add_followers().contacts(i), + more_than_one_follower)); + } + } + if (shorten) { + if (followers.size() > 3) { + // Display 3 names max. + followers.resize(4); + followers[4] = "\u2026"; + } + return Format("Added %s", Join(followers, ", ")); + } else { + if (followers.size() > 1) { + string name_str = Join( + followers.begin(), followers.begin() + followers.size() - 1, ", "); + return Format("added %s and %s", name_str, followers.back()); + } else if (followers.size() == 1) { + return Format("added %s", followers[0]); + } + } + } + + LOG("activity contents not formatted %s", *this); + return ""; +} + +WallTime ActivityTable_Activity::GetViewedTimestamp() const { + // If viewed timestamp isn't set, try reloading it from database. + if (!has_viewed_timestamp()) { + ActivityHandle ah = state_->activity_table()->LoadActivity( + activity_id().local_id(), state_->db()); + return ah->viewed_timestamp(); + } + return viewed_timestamp(); +} + +bool ActivityTable_Activity::IsUpdate() const { + return has_add_followers() || has_update_viewpoint() || has_unshare(); +} + +bool ActivityTable_Activity::IsVisible() const { + // Skip certain uninteresting (or unsupported) activities as well + // as quarantined activities. + if (has_update_viewpoint() || + has_update_episode() || + has_upload_episode() || + has_unshare()) { + return false; + } + // Ignore quarantined activities. + if (label_error()) { + return false; + } + // Ignore empty activities. + if (has_add_followers() && !add_followers().contacts_size()) { + return false; + } + return true; +} + +const ShareEpisodes* ActivityTable_Activity::GetShareEpisodes() { + if (has_share_new()) { + return &share_new().episodes(); + } else if (has_share_existing()) { + return &share_existing().episodes(); + } else if (has_save_photos()) { + return &save_photos().episodes(); + } else if (has_unshare()) { + return &unshare().episodes(); + } + return NULL; +} + +bool ActivityTable_Activity::Load() { + disk_timestamp_ = timestamp(); + return true; +} + +void ActivityTable_Activity::SaveHook(const DBHandle& updates) { + if (viewpoint_id().has_local_id() && has_timestamp()) { + updates->Put(EncodeViewpointActivityKey( + viewpoint_id().local_id(), timestamp(), local_id()), ""); + } + // Build secondary index for episodes added by this activity. + // These are used to build EVENT trapdoors. + if (has_share_new() || has_share_existing()) { + DCHECK(activity_id().has_server_id()); + const ShareEpisodes* episodes = GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + updates->Put(EncodeEpisodeActivityKey( + episodes->Get(i).episode_id().server_id(), local_id()), ""); + // Invalidate the episode if the activity pointing to it changed. + EpisodeHandle eh = state_->episode_table()->LoadEpisode( + episodes->Get(i).episode_id(), updates); + if (eh.get()) { + state_->day_table()->InvalidateEpisode(eh, updates); + } + + if (!provisional()) { + // If the activity is not provisional, mark each of its photos as + // "shared". + const Episode& e = episodes->Get(i); + for (int j = 0; j < e.photo_ids_size(); ++j) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(e.photo_ids(j), updates); + if (ph.get() && !ph->shared()) { + ph->Lock(); + ph->set_shared(true); + ph->SaveAndUnlock(updates); + } + } + } + } + } else if (has_post_comment()) { + // Build secondary index for comment -> activity so we can + // invalidate an activity when the comment is saved. + updates->Put(EncodeCommentActivityKey( + post_comment().comment_id().server_id()), local_id()); + } + if (has_timestamp()) { + updates->Put(EncodeActivityTimestampKey(timestamp(), local_id()), ""); + } + + // If the error label is set, add activity to the unquarantined index. + const string quarantined_key = EncodeQuarantinedActivityKey(local_id()); + if (label_error()) { + updates->Put(quarantined_key, ""); + } else if (updates->Exists(quarantined_key)) { + updates->Delete(quarantined_key); + } + + typedef ContentTable::Content Content; + ActivityHandle ah(reinterpret_cast(this)); + + // Notice if the timestamp has changed and delete old index entries. + if (disk_timestamp_ != timestamp()) { + if (disk_timestamp_ > 0) { + updates->Delete(EncodeViewpointActivityKey( + viewpoint_id().local_id(), disk_timestamp_, local_id())); + updates->Delete(EncodeActivityTimestampKey(disk_timestamp_, local_id())); + + // In the event that the timestamp has changed by more than a + // day, we must invalidate the day table for this activity using + // the prior timestamp. It's easiest if we revert the timestamp + // temporarily and then swap correct value back in. + const double new_timestamp = ah->timestamp(); + ah->set_timestamp(disk_timestamp_); + state_->day_table()->InvalidateActivity(ah, updates); + ah->set_timestamp(new_timestamp); + } + disk_timestamp_ = timestamp(); + } + + // Ugh, ActivityTable_Activity is the base class but ActivityHandle needs a + // pointer to the superclass. + state_->net_queue()->QueueActivity(ah, updates); + + // Invalidate this activity. + state_->day_table()->InvalidateActivity(ah, updates); +} + +void ActivityTable_Activity::DeleteHook(const DBHandle& updates) { + if (viewpoint_id().has_local_id() && has_timestamp()) { + updates->Delete(EncodeViewpointActivityKey( + viewpoint_id().local_id(), timestamp(), local_id())); + } + + if (has_share_new() || has_share_existing()) { + DCHECK(activity_id().has_server_id()); + const ShareEpisodes* episodes = GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + updates->Delete(EncodeEpisodeActivityKey( + episodes->Get(i).episode_id().server_id(), local_id())); + // Delete the episode. + EpisodeHandle eh = state_->episode_table()->LoadEpisode( + episodes->Get(i).episode_id(), updates); + if (eh.get()) { + eh->Lock(); + eh->DeleteAndUnlock(updates); + } + } + } else if (has_post_comment()) { + CommentHandle ch = state_->comment_table()->LoadComment( + post_comment().comment_id(), updates); + if (ch.get()) { + ch->Lock(); + ch->DeleteAndUnlock(updates); + updates->Delete(EncodeCommentActivityKey(ch->comment_id().server_id())); + } + } + if (has_timestamp()) { + updates->Delete(EncodeActivityTimestampKey(timestamp(), local_id())); + } + + // Delete quarantined marker. + updates->Delete(EncodeQuarantinedActivityKey(local_id())); + + // Dequeue and invalidate this activity. + typedef ContentTable::Content Content; + ActivityHandle ah(reinterpret_cast(this)); + state_->net_queue()->DequeueActivity(ah, updates); + state_->day_table()->InvalidateActivity(ah, updates); +} + + +//// +// ActivityIterator + +ActivityTable::ActivityIterator::ActivityIterator( + ActivityTable* table, bool reverse, const DBHandle& db) + : ActivityTable::ContentIterator(db->NewIterator(), reverse), + table_(table), + db_(db), + cur_activity_id_(0), + cur_timestamp_(0) { +} + +ActivityTable::ActivityIterator::~ActivityIterator() { +} + +ActivityHandle ActivityTable::ActivityIterator::GetActivity() { + if (done()) { + return ActivityHandle(); + } + return table_->LoadContent(cur_activity_id_, db_); +} + + +//// +// ActivityTable + +ActivityTable::ActivityTable(AppState* state) + : ContentTable(state, + DBFormat::activity_key(), + DBFormat::activity_server_key(), + kActivityFSCKVersion, + DBFormat::metadata_key("activity_table_fsck")) { +} + +ActivityTable::~ActivityTable() { +} + +ActivityHandle ActivityTable::GetLatestActivity( + int64_t viewpoint_id, const DBHandle& db) { + ScopedPtr vp_iter( + NewViewpointActivityIterator(viewpoint_id, std::numeric_limits::max(), true, db)); + if (!vp_iter->done()) { + return vp_iter->GetActivity(); + } + return ActivityHandle(); +} + +ActivityHandle ActivityTable::GetFirstActivity( + int64_t viewpoint_id, const DBHandle& db) { + ScopedPtr vp_iter( + NewViewpointActivityIterator(viewpoint_id, 0, false, db)); + if (!vp_iter->done()) { + return vp_iter->GetActivity(); + } + return ActivityHandle(); +} + +ActivityHandle ActivityTable::GetCommentActivity( + const string& comment_server_id, const DBHandle& db) { + const string key = EncodeCommentActivityKey(comment_server_id); + const int64_t activity_id = db->Get(key, -1); + if (activity_id != -1) { + return LoadActivity(activity_id, db); + } + return ActivityHandle(); +} + +void ActivityTable::ListEpisodeActivities( + const string& episode_server_id, vector* activity_ids, const DBHandle& db) { + for (DB::PrefixIterator iter(db, EncodeEpisodeActivityKeyPrefix(episode_server_id)); + iter.Valid(); + iter.Next()) { + string dummy_episode_id; + int64_t activity_id; + if (DecodeEpisodeActivityKey(iter.key(), &dummy_episode_id, &activity_id)) { + activity_ids->push_back(activity_id); + } + } +} + +ActivityTable::ActivityIterator* ActivityTable::NewTimestampActivityIterator( + WallTime start, bool reverse, const DBHandle& db) { + return new TimestampActivityIterator(this, start, reverse, db); +} + +ActivityTable::ActivityIterator* ActivityTable::NewViewpointActivityIterator( + int64_t viewpoint_id, WallTime start, bool reverse, const DBHandle& db) { + return new ViewpointActivityIterator(this, viewpoint_id, start, reverse, db); +} + +bool ActivityTable::FSCK( + bool force, ProgressUpdateBlock progress_update, const DBHandle& updates) { + // Restart any quarantined activities. + int unquarantine_count = 0; + for (DB::PrefixIterator iter(updates, kQuarantinedActivityKeyPrefix); + iter.Valid(); + iter.Next()) { + ++unquarantine_count; + int64_t activity_id; + if (DecodeQuarantinedActivityKey(iter.key(), &activity_id)) { + ActivityHandle ah = LoadActivity(activity_id, updates); + LOG("FSCK: unquarantined activity %s", ah->activity_id()); + ah->Lock(); + ah->clear_label_error(); + ah->SaveAndUnlock(updates); + } + } + + LOG("FSCK: unquarantined %d activities", unquarantine_count); + return ContentTable::FSCK(force, progress_update, updates); +} + +bool ActivityTable::FSCKImpl(int prev_fsck_version, const DBHandle& updates) { + LOG("FSCK: ActivityTable"); + bool changes = false; + if (FSCKActivity(updates)) { + changes = true; + } + // Handle any duplicates in secondary indexes by timestamp. These can exist + // as a result of a server bug which rounded up timestamps. + if (FSCKActivityTimestampIndex(updates)) { + changes = true; + } + if (FSCKViewpointActivityIndex(updates)) { + changes = true; + } + return changes; +} + +bool ActivityTable::FSCKActivity(const DBHandle& updates) { + bool changes = false; + for (DB::PrefixIterator iter(updates, DBFormat::activity_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + ActivityMetadata am; + if (am.ParseFromArray(value.data(), value.size())) { + ActivityHandle ah = LoadActivity(am.activity_id().local_id(), updates); + ah->Lock(); + bool save_ah = false; + if (key != EncodeContentKey(DBFormat::activity_key(), am.activity_id().local_id())) { + LOG("FSCK: activity id %d does not equal key %s; deleting key and re-saving", + am.activity_id().local_id(), key); + updates->Delete(key); + save_ah = true; + } + + // Check required fields. + if (!ah->has_activity_id() || + (!ah->has_save_photos() && !ah->has_viewpoint_id()) || + !ah->has_user_id() || + !ah->has_timestamp()) { + LOG("FSCK: activity missing required fields: %s", *ah); + } + + // Check viewpoint; lookup first by server id. + if (ah->viewpoint_id().has_server_id()) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + ah->viewpoint_id().server_id(), updates); + bool invalidate_viewpoint = false; + if (!vh.get()) { + LOG("FSCK: missing viewpoint %s; setting invalidation", ah->viewpoint_id()); + invalidate_viewpoint = true; + } else if (vh->id().local_id() != ah->viewpoint_id().local_id()) { + LOG("FSCK: viewpoint local id mismatch; %d != %d; resetting and clearing secondary indexes", + vh->id().local_id(), ah->viewpoint_id().local_id()); + if (ah->viewpoint_id().has_local_id() && ah->has_timestamp()) { + const string vp_activity_key = EncodeViewpointActivityKey( + ah->viewpoint_id().local_id(), ah->timestamp(), ah->activity_id().local_id()); + updates->Delete(vp_activity_key); + } + if (ah->has_share_new() || ah->has_share_existing()) { + const ShareEpisodes* episodes = ah->GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + const string episode_key = EncodeEpisodeActivityKey( + episodes->Get(i).episode_id().server_id(), ah->activity_id().local_id()); + updates->Delete(episode_key); + } + } + if (ah->has_post_comment()) { + const string comment_activity_key = EncodeCommentActivityKey( + ah->post_comment().comment_id().server_id()); + updates->Delete(comment_activity_key); + } + if (ah->has_timestamp()) { + const string ts_activity_key = EncodeActivityTimestampKey( + ah->timestamp(), ah->activity_id().local_id()); + updates->Delete(ts_activity_key); + } + // Clear quarantined key (if it exists). + const string quarantined_key = + EncodeQuarantinedActivityKey(ah->activity_id().local_id()); + updates->Delete(quarantined_key); + + ah->mutable_viewpoint_id()->clear_local_id(); + save_ah = true; + } else if (ah->update_seq() > vh->update_seq()) { + LOG("FSCK: encountered an activity with update_seq > viewpoint; " + "activity: %s, viewpoint: %s", *ah, *vh); + invalidate_viewpoint = true; + } + + if (invalidate_viewpoint) { + state_->viewpoint_table()->InvalidateFull(ah->viewpoint_id().server_id(), updates); + changes = true; + } + } + if (ah->viewpoint_id().has_server_id() && !ah->viewpoint_id().has_local_id()) { + LOG("FSCK: activity has server id %s but no local id; canonicalizing", + ah->viewpoint_id().server_id()); + state_->viewpoint_table()->CanonicalizeViewpointId(ah->mutable_viewpoint_id(), updates); + save_ah = true; + } + + // Check secondary indexes. + if (ah->viewpoint_id().has_local_id() && ah->has_timestamp()) { + const string vp_activity_key = EncodeViewpointActivityKey( + ah->viewpoint_id().local_id(), ah->timestamp(), ah->activity_id().local_id()); + if (!updates->Exists(vp_activity_key)) { + LOG("FSCK: missing viewpoint activity key"); + save_ah = true; + } + } + if (ah->has_timestamp()) { + const string ts_activity_key = EncodeActivityTimestampKey( + ah->timestamp(), ah->activity_id().local_id()); + if (!updates->Exists(ts_activity_key)) { + LOG("FSCK: missing timestamp activity key"); + save_ah = true; + } + } + if (ah->has_share_new() || ah->has_share_existing()) { + DCHECK(ah->activity_id().has_server_id()); + const ShareEpisodes* episodes = ah->GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + const string episode_key = EncodeEpisodeActivityKey( + episodes->Get(i).episode_id().server_id(), ah->activity_id().local_id()); + if (!updates->Exists(episode_key)) { + LOG("FSCK: missing episode activity key"); + save_ah = true; + } + } + } + if (ah->has_post_comment()) { + const string comment_activity_key = EncodeCommentActivityKey( + ah->post_comment().comment_id().server_id()); + if (!updates->Exists(comment_activity_key)) { + LOG("FSCK: missing comment activity key"); + save_ah = true; + } + } + if (ah->label_error()) { + const string quarantined_key = + EncodeQuarantinedActivityKey(ah->activity_id().local_id()); + if (!updates->Exists(quarantined_key)) { + LOG("FSCK: missing quarantined key for quarantined activity"); + save_ah = true; + } + } + + if (ah->upload_activity() && !ah->provisional()) { + // Re-save any non-uploaded activity to force it to be re-added to + // the network queue. + LOG("FSCK: re-saving non-uploaded activity"); + save_ah = true; + } + + if (save_ah) { + LOG("FSCK: rewriting activity %s", *ah); + ah->SaveAndUnlock(updates); + changes = true; + } else { + ah->Unlock(); + } + } + } + + return changes; +} + +bool ActivityTable::FSCKActivityTimestampIndex(const DBHandle& updates) { + // Map from activity id to secondary index key. + std::unordered_map* activity_ids( + new std::unordered_map); + bool changes = false; + + for (DB::PrefixIterator iter(updates, kActivityTimestampKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + WallTime timestamp; + int64_t activity_id; + if (!DecodeActivityTimestampKey(key, ×tamp, &activity_id)) { + LOG("FSCK: unreadable activity timestamp secondary index: %s", key); + updates->Delete(key); + changes = true; + } else { + if (ContainsKey(*activity_ids, activity_id)) { + LOG("FSCK: activity timestamp secondary index contains duplicate entries for %d; " + "deleting earlier instance (%s)", activity_id, (*activity_ids)[activity_id]); + updates->Delete((*activity_ids)[activity_id]); + changes = true; + } + (*activity_ids)[activity_id] = ToString(key); + } + } + + delete activity_ids; + return changes; +} + +bool ActivityTable::FSCKViewpointActivityIndex(const DBHandle& updates) { + // Map from activity id to secondary index key. + std::unordered_map* activity_ids( + new std::unordered_map); + bool changes = false; + + for (DB::PrefixIterator iter(updates,kViewpointActivityKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + int64_t viewpoint_id; + WallTime timestamp; + int64_t activity_id; + if (!DecodeViewpointActivityKey(key, &viewpoint_id, ×tamp, &activity_id)) { + LOG("FSCK: unreadable activity timestamp secondary index: %s", key); + updates->Delete(key); + changes = true; + } else { + if (ContainsKey(*activity_ids, activity_id)) { + LOG("FSCK: activity timestamp secondary index contains duplicate entries for %d; " + "deleting earlier instance (%s)", activity_id, (*activity_ids)[activity_id]); + updates->Delete((*activity_ids)[activity_id]); + changes = true; + } + (*activity_ids)[activity_id] = ToString(key); + } + } + + delete activity_ids; + return changes; +} + +const ActivityTable::ContactArray* ActivityTable::GetActivityContacts( + const ActivityMetadata& m) { + if (m.has_share_new()) { + return &m.share_new().contacts(); + } + if (m.has_add_followers()) { + return &m.add_followers().contacts(); + } + return NULL; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ActivityTable.h b/clients/shared/ActivityTable.h new file mode 100644 index 0000000..e89ea77 --- /dev/null +++ b/clients/shared/ActivityTable.h @@ -0,0 +1,209 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_ACTIVITY_TABLE_H +#define VIEWFINDER_ACTIVITY_TABLE_H + +#import "ActivityMetadata.pb.h" +#import "ContentTable.h" +#import "DayMetadata.pb.h" +#import "PhotoSelection.h" +#import "WallTime.h" + +typedef google::protobuf::RepeatedPtrField ShareEpisodes; + +// The ActivityTable class maintains the mappings: +// -> +// -> +// ,, -> <> +// , -> <> +// , -> <> +// +// For quarantined activities, maintain an index of device activity ids. +// -> <> + +class ActivityTable_Activity : public ActivityMetadata { + public: + virtual void MergeFrom(const ActivityMetadata& m); + // Unimplemented; exists to get the compiler not to complain about hiding the base class's overloaded MergeFrom. + virtual void MergeFrom(const ::google::protobuf::Message&); + + // Filters the share new or share existing activity of any photos that are + // not present in selection. If the activity is modified, any deletions + // needed to clean up database indexes are added to updates. Returns true if + // a share activity was filtered and it still contains photos. + bool FilterShare(const PhotoSelectionSet& selection, const DBHandle& updates); + + // Returns formatted name of user who created activity. If + // "shorten" is true, returns just first name; otherwise full. + string FormatName(bool shorten); + + // Returns a formatted timestamp, relative to the current date. + string FormatTimestamp(bool shorten); + + // Returns formatted version of activity content. If not NULL, uses the + // supplied activity row to inform the formatting of the activity contents. + // This provides conversation-dependent context, such as eliminating + // photos from a share activity which are duplicates in the conversation. + string FormatContent( + const ViewpointSummaryMetadata::ActivityRow* activity_row, bool shorten); + + // Returns the timestamp at which this activity was viewed. If none + // has been set, returns the current wall time. + WallTime GetViewedTimestamp() const; + + // Returns whether this activity is an update to the conversation, + // as opposed to content. + bool IsUpdate() const; + + // Returns whether the activity is visible in the viewpoint. + // This excludes non-displayed activity types as well as any + // quarantined activities. + bool IsVisible() const; + + // Returns the list of shared episodes, if this is a share_new, + // share_existing, save_photos or unshare activity; NULL otherwise. + const ShareEpisodes* GetShareEpisodes(); + + protected: + bool Load(); + void SaveHook(const DBHandle& updates); + void DeleteHook(const DBHandle& updates); + + // Specialty function for default share activity "caption" in the event + // one is not specified for the activity. + string FormatShareContent( + const ViewpointSummaryMetadata::ActivityRow* activity_row, bool shorten); + + // Invalidates all days which are affected by this activity. This includes + // the day of the activity itself, any days on which episodes shared, updated, + // or unshared in this activity took place, and the first and last days + // of the viewpoint which this activity is part of. + void InvalidateDays(const DBHandle& updates); + + int64_t local_id() const { return activity_id().local_id(); } + const string& server_id() const { return activity_id().server_id(); } + + ActivityTable_Activity(AppState* state, const DBHandle& db, int64_t id); + + protected: + AppState* const state_; + DBHandle db_; + + private: + // The timestamp as stored on disk. + WallTime disk_timestamp_; +}; + +class ActivityTable : public ContentTable { + typedef ActivityTable_Activity Activity; + + typedef ::google::protobuf::RepeatedPtrField ContactArray; + + public: + // Iterates over activities. The current activity + // may be fetched via a call to GetActivity(). + class ActivityIterator : public ContentIterator { + public: + virtual ~ActivityIterator(); + + // Only valid to call if !done(). + ContentHandle GetActivity(); + int64_t activity_id() const { return cur_activity_id_; } + WallTime timestamp() const { return cur_timestamp_; } + + virtual void Seek(WallTime seek_time) = 0; + + protected: + ActivityIterator(ActivityTable* table, bool reverse, const DBHandle& db); + + protected: + ActivityTable* table_; + DBHandle db_; + int64_t cur_activity_id_; + WallTime cur_timestamp_; + }; + + ActivityTable(AppState* state); + virtual ~ActivityTable(); + + ContentHandle NewActivity(const DBHandle& updates) { + return NewContent(updates); + } + ContentHandle LoadActivity(int64_t id, const DBHandle& db) { + return LoadContent(id, db); + } + ContentHandle LoadActivity(const string& server_id, const DBHandle& db) { + return LoadContent(server_id, db); + } + + // Returns the most recent activity for the viewpoint or an + // empty handle if none was found. + ContentHandle GetLatestActivity(int64_t viewpoint_id, const DBHandle& db); + + // Returns the first activity for the viewpoint or an empty + // handle if none was found. + ContentHandle GetFirstActivity(int64_t viewpoint_id, const DBHandle& db); + + // Returns the activity that posted the comment. + ContentHandle GetCommentActivity(const string& comment_server_id, const DBHandle& db); + + // Returns the activities which added photos from the specified episode. + void ListEpisodeActivities( + const string& episode_server_id, vector* activity_ids, const DBHandle& db); + + // Returns a new ActivityIterator object for iterating over + // activities in timestamp order. Specify reverse to iterate from + // most to least recent. The caller is responsible for deleting the + // iterator. + ActivityIterator* NewTimestampActivityIterator( + WallTime start, bool reverse, const DBHandle& db); + + // Returns a new ActivityIterator object for iterating over + // activities in the specified viewpoint. The activities within the + // viewpoint are returned in sorted timestamp order from oldest to + // newest. Specify reverse to iterate instead from newest to + // oldest. The caller is responsible for deleting the iterator. + ActivityIterator* NewViewpointActivityIterator( + int64_t viewpoint_id, WallTime start, bool reverse, const DBHandle& db); + + // Override of base class FSCK to unquarantine activities on startup. + virtual bool FSCK( + bool force, ProgressUpdateBlock progress_update, const DBHandle& updates); + + // Repairs secondary indexes and sanity checks all references from + // activities to other assets. + bool FSCKImpl(int prev_fsck_version, const DBHandle& updates); + + // Consistency check on activity metdata. + bool FSCKActivity(const DBHandle& updates); + + // Consistency check on activity-by-timestamp index. + bool FSCKActivityTimestampIndex(const DBHandle& updates); + + // Consistency check on viewpoint-activity-timestamp index. + bool FSCKViewpointActivityIndex(const DBHandle& updates); + + static const ContactArray* GetActivityContacts(const ActivityMetadata& m); +}; + +typedef ActivityTable::ContentHandle ActivityHandle; + +string EncodeActivityTimestampKey(WallTime timestamp, int64_t activity_id); +string EncodeCommentActivityKey(const string& episode_server_id); +string EncodeEpisodeActivityKey(const string& episode_server_id, int64_t activity_id); +string EncodeEpisodeActivityKeyPrefix(const string& episode_server_id); +string EncodeQuarantinedActivityKey(int64_t activity_id); +string EncodeViewpointActivityKey(int64_t viewpoint_id, WallTime timestamp, int64_t activity_id); +bool DecodeActivityTimestampKey(Slice key, WallTime* timestamp, int64_t* activity_id); +bool DecodeCommentActivityKey(Slice key, string* comment_server_id); +bool DecodeEpisodeActivityKey(Slice key, string* episode_server_id, int64_t* activity_id); +bool DecodeViewpointActivityKey(Slice key, int64_t* viewpoint_id, + WallTime* timestamp, int64_t* activity_id); +bool DecodeQuarantinedActivityKey(Slice key, int64_t* activity_id); + +#endif // VIEWFINDER_ACTIVITY_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Analytics.cc b/clients/shared/Analytics.cc new file mode 100644 index 0000000..2a4b31c --- /dev/null +++ b/clients/shared/Analytics.cc @@ -0,0 +1,871 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "Analytics.h" +#import "ContactMetadata.pb.h" +#import "FileUtils.h" +#import "IdentityManager.h" +#import "Logging.h" +#import "PathUtils.h" + +// Shorthand for simple events with no parameters. +#define EVENT(meth_name, event_name) \ + void Analytics::meth_name() { \ + TrackEvent(event_name); \ + } + +namespace { + +void GetContactProperties(const ContactMetadata& contact, JsonDict* d) { + d->insert("has_user_id", contact.has_user_id()); + d->insert("registered", contact.label_registered()); + d->insert("identities", contact.identities_size()); + + int email = 0; + int phone = 0; + int facebook = 0; + for (int i = 0; i < contact.identities_size(); i++) { + const string& identity = contact.identities(i).identity(); + if (IdentityManager::IsEmailIdentity(identity)) { + email++; + } else if (IdentityManager::IsPhoneIdentity(identity)) { + phone++; + } else if (IdentityManager::IsFacebookIdentity(identity)) { + facebook++; + } + } + if (email) { + d->insert("email_identities", email); + } + if (phone) { + d->insert("phone_identities", phone); + } + if (facebook) { + d->insert("facebook_identities", facebook); + } +} + +} // namespace + +Analytics::Analytics(bool enabled) + : enabled_(enabled), + fd_(-1), + count_(0) { +} + +Analytics::~Analytics() { +} + +void Analytics::Launch(const string& reason) { + TrackEvent("/app-state/launch", + JsonDict("reason", reason)); +} + +void Analytics::EnterBackground() { + TrackEvent("/app-state/background"); +} + +void Analytics::EnterForeground() { + TrackEvent("/app-state/foreground"); +} + +void Analytics::RemoteNotification() { + TrackEvent("/app-state/remote_notification"); +} + +void Analytics::SummaryDashboard() { + TrackEvent("/ui/summary/dashboard"); +} + +void Analytics::SummaryLibrary() { + TrackEvent("/ui/summary/events"); +} + +void Analytics::SummaryInbox() { + TrackEvent("/ui/summary/inbox"); +} + +void Analytics::SummaryViewfinder(const string& page, WallTime duration) { + TrackEvent("/ui/summary/viewfinder", + JsonDict({ + { "duration", duration }, + { "page", page } + })); +} + +void Analytics::DashboardMyInfoButton() { + TrackEvent("/ui/dashboard/my_info"); +} + +void Analytics::DashboardContactsButton() { + TrackEvent("/ui/dashboard/contacts"); +} + +void Analytics::DashboardSettingsButton() { + TrackEvent("/ui/dashboard/settings"); +} + +void Analytics::DashboardPhotoCount() { + TrackEvent("/ui/dashboard/photo_count"); +} + +void Analytics::DashboardContactCount() { + TrackEvent("/ui/dashboard/contact_count"); +} + +void Analytics::DashboardConversationCount() { + TrackEvent("/ui/dashboard/conversation_count"); +} + +void Analytics::DashboardNoticeExpand(int type) { + TrackEvent("/ui/dashboard/notice_expand", JsonDict("type", type)); +} + +void Analytics::DashboardNoticeClick(int type) { + TrackEvent("/ui/dashboard/notice_click", JsonDict("type", type)); +} + +void Analytics::DashboardNoticeDismiss(int type) { + TrackEvent("/ui/dashboard/notice_dismiss", JsonDict("type", type)); +} + +void Analytics::EventPage(WallTime timestamp) { + TrackEvent("/ui/event", JsonDict("timestamp", int(timestamp))); +} + +void Analytics::EventViewfinder(WallTime duration) { + TrackEvent("/ui/event/viewfinder", JsonDict("duration", duration)); +} + +void Analytics::EventShareNew(int num_photos, int num_contacts) { + TrackEvent("/ui/event/share_new", + JsonDict({ + { "photos", num_photos }, + { "contacts", num_contacts } + })); +} + +void Analytics::EventShareExisting(int num_photos) { + TrackEvent("/ui/event/share_existing", + JsonDict("photos", num_photos)); +} + +void Analytics::EventAddFollowers(int num_contacts) { + TrackEvent("/ui/event/add_followers", + JsonDict("contacts", num_contacts)); +} + +void Analytics::EventRemovePhotos(int num_photos) { + TrackEvent("/ui/event/delete", JsonDict("photos", num_photos)); +} + +void Analytics::EventExport(int num_photos) { + TrackEvent("/ui/event/export", JsonDict("photos", num_photos)); +} + +void Analytics::EventExpand() { + TrackEvent("/ui/event/expand"); +} + +void Analytics::EventSearchButton() { + TrackEvent("/ui/event/search"); +} + +void Analytics::EventCameraButton() { + TrackEvent("/ui/event/camera"); +} + +void Analytics::EventShareButton() { + TrackEvent("/ui/event/share"); +} + +void Analytics::EventActionToggle() { + TrackEvent("/ui/event/action"); +} + +void Analytics::InboxCardExpand() { + TrackEvent("/ui/inbox/card_expand"); +} + +void Analytics::InboxSearchButton() { + TrackEvent("/ui/inbox/search"); +} + +void Analytics::InboxCameraButton() { + TrackEvent("/ui/inbox/camera"); +} + +void Analytics::InboxShareButton() { + TrackEvent("/ui/inbox/share"); +} + +void Analytics::InboxActionToggle() { + TrackEvent("/ui/inbox/action"); +} + +void Analytics::ConversationPage(int64_t viewpoint_id) { + TrackEvent("/ui/conversation", JsonDict("viewpoint_id", viewpoint_id)); +} + +void Analytics::ConversationAutosaveOff() { + TrackEvent("/ui/conversation/autosave_off"); +} + +void Analytics::ConversationAutosaveOn() { + TrackEvent("/ui/conversation/autosave_on"); +} + +void Analytics::ConversationLeave() { + TrackEvent("/ui/conversation/leave"); +} + +void Analytics::ConversationRemove() { + TrackEvent("/ui/conversation/remove"); +} + +void Analytics::ConversationMute() { + TrackEvent("/ui/conversation/mute"); +} + +void Analytics::ConversationUnmute() { + TrackEvent("/ui/conversation/unmute"); +} + +void Analytics::ConversationViewfinder(WallTime duration) { + TrackEvent("/ui/conversation/viewfinder", JsonDict("duration", duration)); +} + +void Analytics::ConversationRemovePhotos(int num_photos) { + TrackEvent("/ui/conversation/remove_photos", + JsonDict("photos", num_photos)); +} + +void Analytics::ConversationSavePhotos(int num_photos) { + TrackEvent("/ui/conversation/save_photos", + JsonDict("photos", num_photos)); +} + +void Analytics::ConversationShareNew(int num_photos, int num_contacts) { + TrackEvent("/ui/conversation/share_new", + JsonDict({ + { "photos", num_photos }, + { "contacts", num_contacts } + })); +} + +void Analytics::ConversationShareExisting(int num_photos) { + TrackEvent("/ui/conversation/share_existing", + JsonDict("photos", num_photos)); +} + +void Analytics::ConversationAddFollowers(int num_contacts) { + TrackEvent("/ui/conversation/add_followers", + JsonDict("contacts", num_contacts)); +} + +void Analytics::ConversationRemoveFollowers(int num_followers) { + TrackEvent("/ui/conversation/remove_followers", + JsonDict("followers", num_followers)); +} + +void Analytics::ConversationUnshare(int num_photos) { + TrackEvent("/ui/conversation/unshare", JsonDict("photos", num_photos)); +} + +void Analytics::ConversationExport(int num_photos) { + TrackEvent("/ui/conversation/export", JsonDict("photos", num_photos)); +} + +void Analytics::ConversationCameraButton() { + TrackEvent("/ui/conversation/camera"); +} + +void Analytics::ConversationAddPhotosButton() { + TrackEvent("/ui/conversation/add_photos"); +} + +void Analytics::ConversationAddPeopleButton() { + TrackEvent("/ui/conversation/add_people"); +} + +void Analytics::ConversationEditToggle() { + TrackEvent("/ui/conversation/edit"); +} + +void Analytics::ConversationSelectFollowerGroup(int size) { + TrackEvent("/ui/conversation/select_follower_group", JsonDict("size", size)); +} + +void Analytics::ConversationSelectFollower(const ContactMetadata& contact) { + JsonDict props; + GetContactProperties(contact, &props); + TrackEvent("/ui/conversation/select_follower", props); +} + +void Analytics::ConversationSelectFollowerIdentity(const ContactMetadata& contact, const string& identity) { + JsonDict props; + GetContactProperties(contact, &props); + props.insert("selected_identity", IdentityManager::IdentityType(identity)); + TrackEvent("/ui/conversation/select_follower_identity", props); +} + +void Analytics::ConversationUpdateCoverPhoto() { + TrackEvent("/ui/conversation/update_cover_photo"); +} + +void Analytics::ConversationUpdateTitle() { + TrackEvent("/ui/conversation/update_title"); +} + +EVENT(OnboardingStart, "/ui/onboarding/start"); +EVENT(OnboardingSignupCard, "/ui/onboarding/signup_card"); +EVENT(OnboardingLoginCard, "/ui/onboarding/login_card"); +EVENT(OnboardingResetPasswordCard, "/ui/onboarding/reset_password_card"); +EVENT(OnboardingLinkCard, "/ui/onboarding/link_card"); +EVENT(OnboardingMergeCard, "/ui/onboarding/merge_card"); +EVENT(OnboardingSetPasswordCard, "/ui/onboarding/set_password_card"); +EVENT(OnboardingChangePasswordCard, "/ui/onboarding/change_password_card"); +EVENT(OnboardingConfirmCard, "/ui/onboarding/confirm_card"); +EVENT(OnboardingResetDeviceIdCard, "/ui/onboarding/reset_device_id_card"); + +EVENT(OnboardingError, "/ui/onboarding/error"); +EVENT(OnboardingNetworkError, "/ui/onboarding/network_error"); +EVENT(OnboardingCancel, "/ui/onboarding/cancel"); +EVENT(OnboardingChangePasswordComplete, "/ui/onboarding/change_password_complete"); +EVENT(OnboardingResendCode, "/ui/onboarding/resend_code"); +EVENT(OnboardingConfirmEmail, "/ui/onboarding/confirm_email"); +EVENT(OnboardingConfirmPhone, "/ui/onboarding/confirm_phone"); +EVENT(OnboardingConfirmComplete, "/ui/onboarding/confirm_complete"); +EVENT(OnboardingConfirmEmailComplete, "/ui/onboarding/confirm_email_complete"); +EVENT(OnboardingConfirmPhoneComplete, "/ui/onboarding/confirm_phone_complete"); + +EVENT(OnboardingAccountSetupCard, "/ui/onboarding/account_setup_card"); +EVENT(OnboardingImportContacts, "/ui/onboarding/import_contacts"); +EVENT(OnboardingSkipImportContacts, "/ui/onboarding/skip_import_contacts"); +EVENT(OnboardingComplete, "/ui/onboarding/complete"); + +void Analytics::PhotoExport() { + TrackEvent("/ui/photo/export", JsonDict("photos", 1)); +} + +void Analytics::PhotoPage(int photo_id) { + TrackEvent("/ui/photo", JsonDict("photo_id", photo_id)); +} + +void Analytics::PhotoShareNew(int num_contacts) { + TrackEvent("/ui/photo/share_new", + JsonDict({ + { "photos", 1 }, + { "contacts", num_contacts } + })); +} + +void Analytics::PhotoShareExisting() { + TrackEvent("/ui/photo/share_existing", + JsonDict("photos", 1)); +} + +void Analytics::PhotoAddFollowers(int num_contacts) { + TrackEvent("/ui/photo/add_followers", + JsonDict("contacts", num_contacts)); +} + +void Analytics::PhotoRemove() { + TrackEvent("/ui/photo/delete", JsonDict("photos", 1)); +} + +void Analytics::PhotoSave() { + TrackEvent("/ui/photo/save", JsonDict("photos", 1)); +} + +void Analytics::PhotoUnshare() { + TrackEvent("/ui/photo/unshare", JsonDict("photos", 1)); +} + +void Analytics::PhotoToolbarToggle(bool on) { + TrackEvent("/ui/photo/toolbar_toggle", JsonDict("state", on)); +} + +void Analytics::PhotoZoom() { + TrackEvent("/ui/photo/zoom"); +} + +void Analytics::PhotoSwipeDismiss() { + TrackEvent("/ui/photo/swipe_dismiss"); +} + +void Analytics::CameraPage(const string& type) { + TrackEvent("/ui/camera", + JsonDict("type", ToLowercase(type))); +} + +void Analytics::CameraTakePicture() { + TrackEvent("/ui/camera/take-picture"); +} + +void Analytics::CameraFlashOn() { + TrackEvent("/ui/camera/flash-on"); +} + +void Analytics::CameraFlashOff() { + TrackEvent("/ui/camera/flash-off"); +} + +void Analytics::CameraFlashAuto() { + TrackEvent("/ui/camera/flash-auto"); +} + +void Analytics::ContactsPage() { + TrackEvent("/ui/contacts"); +} + +void Analytics::ContactsSourceToggle(bool all) { + TrackEvent("/ui/contacts/source_toggle", JsonDict("all", all)); +} + +void Analytics::ContactsSearch() { + TrackEvent("/ui/contacts/search"); +} + +void Analytics::AddContactsPage() { + TrackEvent("/ui/add_contacts"); +} + +void Analytics::AddContactsManualTyping() { + TrackEvent("/ui/add_contacts/manual/typing"); +} + +void Analytics::AddContactsManualComplete() { + TrackEvent("/ui/add_contacts/manual/complete"); +} + +void Analytics::ContactsFetch(const string& service) { + TrackEvent(Format("/ui/contacts/fetch/%s", service)); +} + +void Analytics::ContactsFetchComplete(const string& service) { + TrackEvent(Format("/ui/contacts/fetch_complete/%s", service)); +} + +void Analytics::ContactsFetchError(const string& service, const string& reason) { + TrackEvent(Format("/ui/contacts/fetch_complete/%s", service), JsonDict("reason", reason)); +} + +void Analytics::ContactsRefresh(const string& service) { + TrackEvent(Format("/ui/contacts/refresh/%s", service)); +} + +void Analytics::ContactsRemove(const string& service) { + TrackEvent(Format("/ui/contacts/remove/%s", service)); +} + +void Analytics::AddContactsLinkPhoneStart() { + TrackEvent("/ui/add_contacts/link_phone/start"); +} + +void Analytics::AddContactsLinkPhoneComplete() { + TrackEvent("/ui/add_contacts/link_phone/start"); +} + +void Analytics::ContactInfoPage(bool me) { + TrackEvent("/ui/contact_info", JsonDict("me", me)); +} + +void Analytics::ContactInfoShowConversations() { + TrackEvent("/ui/contact_info/show_conversations"); +} + +void Analytics::ContactInfoStartConversation() { + TrackEvent("/ui/contact_info/start_conversation"); +} + +void Analytics::ContactInfoAddIdentity() { + TrackEvent("/ui/contact_info/add_identity"); +} + +void Analytics::ContactInfoChangePassword() { + TrackEvent("/ui/contact_info/change_password"); +} + +void Analytics::SettingsCloudStorage() { + TrackEvent("/ui/settings/cloud_storage"); +} + +void Analytics::SettingsLogin() { + TrackEvent("/ui/settings/login"); +} + +void Analytics::SettingsPage() { + TrackEvent("/ui/settings"); +} + +void Analytics::SettingsPrivacyPolicy() { + TrackEvent("/ui/settings/privacy_policy"); +} + +void Analytics::SettingsRegister() { + TrackEvent("/ui/settings/register"); +} + +void Analytics::SettingsTermsOfService() { + TrackEvent("/ui/settings/terms_of_service"); +} + +void Analytics::SettingsUnlink() { + TrackEvent("/ui/settings/unlink"); +} + +void Analytics::SendFeedback(int result) { + TrackEvent("/ui/send_feedback", JsonDict("result", result)); +} + +void Analytics::Network(bool up, bool wifi) { + TrackEvent("/network/state", + JsonDict({ + { "up", up }, + { "wifi", wifi } + })); +} + +void Analytics::NetworkAddFollowers(int status, WallTime elapsed) { + TrackEvent("/network/add_followers", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkAuthViewfinder(int status, WallTime elapsed) { + TrackEvent("/network/auth_viewfinder", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkBenchmarkDownload(int status, bool up, bool wifi, const string& url, + int64_t bytes, WallTime elapsed) { + TrackEvent("/network/benchmark_download", + JsonDict({ + { "status", status }, + { "elapsed", elapsed }, + { "up", up }, + { "wifi", wifi }, + { "url", url }, + { "bytes", bytes } + })); +} + +void Analytics::NetworkDownloadPhoto(int status, int64_t bytes, const string& type, WallTime elapsed) { + TrackEvent("/network/download_photo", + JsonDict({ + { "status", status }, + { "elapsed", elapsed }, + { "bytes", bytes }, + { "type", type } + })); +} + +void Analytics::NetworkFetchFacebookContacts(int status, WallTime elapsed) { + TrackEvent("/network/fetch_facebook_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkFetchGoogleContacts(int status, WallTime elapsed) { + TrackEvent("/network/fetch_google_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkLinkIdentity(int status, WallTime elapsed) { + TrackEvent("/network/link_identity", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkMergeAccounts(int status, WallTime elapsed) { + TrackEvent("/network/merge_accounts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkPing(int status, WallTime elapsed) { + TrackEvent("/network/ping", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkPostComment(int status, WallTime elapsed) { + TrackEvent("/network/post_comment", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryContacts(int status, WallTime elapsed) { + TrackEvent("/network/query_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryEpisodes(int status, WallTime elapsed) { + TrackEvent("/network/query_episodes", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryFollowed(int status, WallTime elapsed) { + TrackEvent("/network/query_followed", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryNotifications(int status, WallTime elapsed) { + TrackEvent("/network/query_notifications", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryUsers(int status, WallTime elapsed) { + TrackEvent("/network/query_users", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkQueryViewpoints(int status, WallTime elapsed) { + TrackEvent("/network/query_viewpoints", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkRecordSubscription(int status, WallTime elapsed) { + TrackEvent("/network/record_subscription", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkRemoveContacts(int status, WallTime elapsed) { + TrackEvent("/network/remove_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkRemoveFollowers(int status, WallTime elapsed) { + TrackEvent("/network/remove_followers", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkRemovePhotos(int status, WallTime elapsed) { + TrackEvent("/network/remove_photos", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkResolveContacts(int status, WallTime elapsed) { + TrackEvent("/network/resolve_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkSavePhotos(int status, WallTime elapsed) { + TrackEvent("/network/save_photos", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkShare(int status, WallTime elapsed) { + TrackEvent("/network/share", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUnshare(int status, WallTime elapsed) { + TrackEvent("/network/unshare", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUpdateDevice(int status, WallTime elapsed) { + TrackEvent("/network/update_device", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUpdateFriend(int status, WallTime elapsed) { + TrackEvent("/network/update_friend", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUpdateUser(int status, WallTime elapsed) { + TrackEvent("/network/update_user", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUpdateUserPhoto(int status, WallTime elapsed) { + TrackEvent("/network/update_user_photo", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUpdateViewpoint(int status, WallTime elapsed) { + TrackEvent("/network/update_viewpoint", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUploadContacts(int status, WallTime elapsed) { + TrackEvent("/network/upload_contacts", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUploadEpisode(int status, WallTime elapsed) { + TrackEvent("/network/upload_episode", + JsonDict({ + { "status", status }, + { "elapsed", elapsed } + })); +} + +void Analytics::NetworkUploadPhoto(int status, int64_t bytes, const string& type, WallTime elapsed) { + TrackEvent("/network/upload_photo", + JsonDict({ + { "status", status }, + { "elapsed", elapsed }, + { "bytes", bytes }, + { "type", type } + })); +} + +void Analytics::NetworkVerifyViewfinder(int status, WallTime elapsed, bool manual_entry) { + TrackEvent("/network/verify_viewfinder", + JsonDict({ + { "status", status }, + { "elapsed", elapsed }, + { "manual_entry", manual_entry } + })); +} + +void Analytics::AssetsScan(bool full_scan, int num_assets, + int num_scanned, WallTime elapsed) { + TrackEvent("/assets/scan", + JsonDict({ + { "type", full_scan ? "full" : "fast" }, + { "num_assets", num_assets }, + { "num_scanned", num_scanned }, + { "elapsed", elapsed } + })); +} + +void Analytics::LocalUsage(int64_t bytes, int thumbnail_files, + int medium_files, int full_files, + int original_files) { + TrackEvent("/storage/usage", + JsonDict({ + { "bytes", bytes }, + { "thumbnail_files", thumbnail_files }, + { "medium_files", medium_files }, + { "full_files", full_files }, + { "original_files", original_files } + })); +} + +void Analytics::TrackEvent(const string& name, JsonDict properties) { + if (!enabled_) { + return; + } + + const string e = FormatEntry(name, properties); + + MutexLock l(&mu_); + + if (fd_ < 0) { + const string log_filename = NewLogFilename(".analytics"); + const string log_path = JoinPath(LoggingDir(), log_filename); + fd_ = FileCreate(log_path); + if (fd_ < 0) { + LOG("init: unable to open: %s: %d (%s)", + log_path, errno, strerror(errno)); + return; + } + WriteStringToFD(fd_, "["); + } + + if (count_ > 0) { + WriteStringToFD(fd_, ",\n"); + } + + WriteStringToFD(fd_, e); + ++count_; +} + +string Analytics::FormatEntry(const string& name, JsonDict properties) { + properties.insert("name", name); + properties.insert("timestamp", WallTime_Now()); + string s = properties.FormatCompact(); + if (Slice(s).ends_with("\n")) { + // Strip the trailing newline added by FormatCompact(). + s.resize(s.size() - 1); + } + return s; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Analytics.h b/clients/shared/Analytics.h new file mode 100644 index 0000000..fadbb7c --- /dev/null +++ b/clients/shared/Analytics.h @@ -0,0 +1,229 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "JsonUtils.h" +#import "Mutex.h" +#import "WallTime.h" + +class ContactMetadata; + +class Analytics { + public: + explicit Analytics(bool enabled); + ~Analytics(); + + //// + //// TODO(peter): get rid of these boilerplate methods and call + //// TrackEvent directly instead. + //// + + // App state. + void Launch(const string& reason); + void EnterBackground(); + void EnterForeground(); + void RemoteNotification(); + + // Summary page. + void SummaryDashboard(); + void SummaryLibrary(); + void SummaryInbox(); + void SummaryViewfinder(const string& page, WallTime duration); + + // Dashboard page. + void DashboardMyInfoButton(); + void DashboardContactsButton(); + void DashboardSettingsButton(); + void DashboardPhotoCount(); + void DashboardContactCount(); + void DashboardConversationCount(); + void DashboardNoticeExpand(int type); + void DashboardNoticeClick(int type); + void DashboardNoticeDismiss(int type); + + // Event page. + void EventPage(WallTime timestamp); + void EventViewfinder(WallTime duration); + void EventShareNew(int num_photos, int num_contacts); + void EventShareExisting(int num_photos); + void EventAddFollowers(int num_contacts); + void EventRemovePhotos(int num_photos); + void EventExport(int num_photos); + void EventExpand(); + void EventSearchButton(); + void EventCameraButton(); + void EventShareButton(); + void EventActionToggle(); + + // Inbox page. + void InboxCardExpand(); + void InboxSearchButton(); + void InboxCameraButton(); + void InboxShareButton(); + void InboxActionToggle(); + + // Conversation page. + void ConversationPage(int64_t viewpoint_id); + void ConversationAutosaveOff(); + void ConversationAutosaveOn(); + void ConversationLeave(); + void ConversationRemove(); + void ConversationMute(); + void ConversationUnmute(); + void ConversationViewfinder(WallTime duration); + void ConversationRemovePhotos(int num_photos); + void ConversationSavePhotos(int num_photos); + void ConversationShareNew(int num_photos, int num_contacts); + void ConversationShareExisting(int num_photos); + void ConversationAddFollowers(int num_contacts); + void ConversationRemoveFollowers(int num_followers); + void ConversationUnshare(int num_photos); + void ConversationExport(int num_photos); + void ConversationCameraButton(); + void ConversationAddPhotosButton(); + void ConversationAddPeopleButton(); + void ConversationEditToggle(); + void ConversationSelectFollowerGroup(int size); + void ConversationSelectFollower(const ContactMetadata& contact); + void ConversationSelectFollowerIdentity(const ContactMetadata& contact, const string& identity); + void ConversationUpdateCoverPhoto(); + void ConversationUpdateTitle(); + + // Onboarding pages. + void OnboardingStart(); + void OnboardingSignupCard(); + void OnboardingLoginCard(); + void OnboardingResetPasswordCard(); + void OnboardingLinkCard(); + void OnboardingMergeCard(); + void OnboardingSetPasswordCard(); + void OnboardingChangePasswordCard(); + void OnboardingConfirmCard(); + void OnboardingResetDeviceIdCard(); + + void OnboardingError(); + void OnboardingNetworkError(); + void OnboardingCancel(); + void OnboardingChangePasswordComplete(); + void OnboardingResendCode(); + void OnboardingConfirmEmail(); + void OnboardingConfirmPhone(); + void OnboardingConfirmComplete(); + void OnboardingConfirmEmailComplete(); + void OnboardingConfirmPhoneComplete(); + + void OnboardingAccountSetupCard(); + void OnboardingImportContacts(); + void OnboardingSkipImportContacts(); + void OnboardingComplete(); + + // Photo page. + void PhotoExport(); + void PhotoPage(int photo_id); + void PhotoShareNew(int num_contacts); + void PhotoShareExisting(); + void PhotoAddFollowers(int num_contacts); + void PhotoRemove(); + void PhotoSave(); + void PhotoUnshare(); + void PhotoToolbarToggle(bool on); + void PhotoZoom(); + void PhotoSwipeDismiss(); + + // Camera page. + void CameraPage(const string& type); + void CameraTakePicture(); + void CameraFlashOn(); + void CameraFlashOff(); + void CameraFlashAuto(); + + // Contacts page. + void ContactsPage(); + void ContactsSourceToggle(bool all); + void ContactsSearch(); + + // Add contacts page. + void AddContactsPage(); + void AddContactsManualTyping(); + void AddContactsManualComplete(); + void ContactsFetch(const string& service); + void ContactsFetchComplete(const string& service); + void ContactsFetchError(const string& service, const string& reason); + void ContactsRefresh(const string& service); + void ContactsRemove(const string& service); + void AddContactsLinkPhoneStart(); + void AddContactsLinkPhoneComplete(); + + // Contact info page. + void ContactInfoPage(bool me); + void ContactInfoShowConversations(); + void ContactInfoStartConversation(); + void ContactInfoAddIdentity(); + void ContactInfoChangePassword(); + + // Settings page. + void SettingsCloudStorage(); + void SettingsLogin(); + void SettingsPage(); + void SettingsPrivacyPolicy(); + void SettingsRegister(); + void SettingsTermsOfService(); + void SettingsUnlink(); + + // Send feedback. + void SendFeedback(int result); + + // Network. + void Network(bool up, bool wifi); + void NetworkAddFollowers(int status, WallTime elapsed); + void NetworkAuthViewfinder(int status, WallTime elapsed); + void NetworkBenchmarkDownload(int status, bool up, bool wifi, const string& url, int64_t bytes, WallTime elapsed); + void NetworkDownloadPhoto(int status, int64_t bytes, const string& type, WallTime elapsed); + void NetworkFetchFacebookContacts(int status, WallTime elapsed); + void NetworkFetchGoogleContacts(int status, WallTime elapsed); + void NetworkLinkIdentity(int status, WallTime elapsed); + void NetworkMergeAccounts(int status, WallTime elapsed); + void NetworkPing(int status, WallTime elapsed); + void NetworkPostComment(int status, WallTime elapsed); + void NetworkQueryContacts(int status, WallTime elapsed); + void NetworkQueryEpisodes(int status, WallTime elapsed); + void NetworkQueryFollowed(int status, WallTime elapsed); + void NetworkQueryNotifications(int status, WallTime elapsed); + void NetworkQueryUsers(int status, WallTime elapsed); + void NetworkQueryViewpoints(int status, WallTime elapsed); + void NetworkRecordSubscription(int status, WallTime elapsed); + void NetworkRemoveContacts(int status, WallTime elapsed); + void NetworkRemoveFollowers(int status, WallTime elapsed); + void NetworkRemovePhotos(int status, WallTime elapsed); + void NetworkResolveContacts(int status, WallTime elapsed); + void NetworkSavePhotos(int status, WallTime elapsed); + void NetworkShare(int status, WallTime elapsed); + void NetworkUnshare(int status, WallTime elapsed); + void NetworkUpdateDevice(int status, WallTime elapsed); + void NetworkUpdateFriend(int status, WallTime elapsed); + void NetworkUpdateUser(int status, WallTime elapsed); + void NetworkUpdateUserPhoto(int status, WallTime elapsed); + void NetworkUpdateViewpoint(int status, WallTime elapsed); + void NetworkUploadContacts(int status, WallTime elapsed); + void NetworkUploadEpisode(int status, WallTime elapsed); + void NetworkUploadPhoto(int status, int64_t bytes, const string& type, WallTime elapsed); + void NetworkVerifyViewfinder(int status, WallTime elapsed, bool manual_entry); + + // Assets. + void AssetsScan(bool full_scan, int num_assets, + int num_scanned, WallTime elapsed); + + // Local storage. + void LocalUsage(int64_t bytes, int thumbnail_files, + int medium_files, int full_files, + int original_files); + + private: + void TrackEvent(const string& name, JsonDict properties = JsonDict()); + string FormatEntry(const string& name, JsonDict properties); + + private: + const bool enabled_; + Mutex mu_; + int fd_; + int count_; +}; diff --git a/clients/shared/AppState.cc b/clients/shared/AppState.cc new file mode 100644 index 0000000..f8a7ead --- /dev/null +++ b/clients/shared/AppState.cc @@ -0,0 +1,607 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "AppState.h" +#import "Analytics.h" +#import "AsyncState.h" +#import "Breadcrumb.pb.h" +#import "CommentTable.h" +#import "ContactManager.h" +#import "DayTable.h" +#import "DBStats.h" +#import "Defines.h" +#import "FileUtils.h" +#import "GeocodeManager.h" +#import "ImageIndex.h" +#import "NetworkManager.h" +#import "NetworkQueue.h" +#import "NotificationManager.h" +#import "PathUtils.h" +#import "PeopleRank.h" +#import "PhotoStorage.h" +#import "PlacemarkHistogram.h" +#import "PlacemarkTable.h" +#import "Timer.h" + +namespace { + +#ifndef DB_FORMAT_VALUE +#define DB_FORMAT_VALUE "56" +#endif // DB_FORMAT_VALUE + +#ifndef RESET_STATE +#define RESET_STATE false +#endif // RESET_STATE + +const bool kResetState = RESET_STATE; + +// The major format of the database. If the major format does not match, the +// database and all associated data (including photos!) is destroyed. +const string kFormatKey = DBFormat::metadata_key("format"); +const string kFormatValue = DB_FORMAT_VALUE; + +const int kCacheSize = 1 * 1024 * 1024; + +const string kInitMaintenanceKey = DBFormat::metadata_key("init_maintenance"); +const string kServerHostKey = DBFormat::metadata_key("server_host"); +const string kDeviceIdKey = DBFormat::metadata_key("device_id"); +const string kUserIdKey = DBFormat::metadata_key("user_id"); +const string kUserCookieKey = DBFormat::metadata_key("user_cookie"); +const string kXsrfCookieKey = DBFormat::metadata_key("xsrf_cookie"); +const string kCloudStorageKey = DBFormat::metadata_key("cloud_storage"); +const string kStoreOriginalsKey = DBFormat::metadata_key("store_originals"); +const string kNoPasswordKey = DBFormat::metadata_key("no_password"); +const string kRefreshCompletedKey = DBFormat::metadata_key("refresh_completed"); +const string kUploadLogsKey = DBFormat::metadata_key("upload_logs"); +const string kLastLoginTimestampKey = DBFormat::metadata_key("last_login_timestamp"); +const string kRegistrationVersionKey = DBFormat::metadata_key("registration_version"); +const string kSystemMessageKey = DBFormat::metadata_key("system_message"); +const string kLastBreadcrumbKey = DBFormat::metadata_key("last_breadcrumb"); + +// Maintains a sequence of local operation ids. The ids from this +// sequence are combined with the device id to encode a server-side +// operation id, passed with mutating requests (share, unshare, +// upload, delete, remove, etc.). The operation id (and an associated +// timestamp) are included in the JSON request "headers" dict. The +// activity which is subsequently generated for that operation will be +// composed exactly of the op timestamp, device id, and local +// operation id. +const string kNextOperationIdKey = DBFormat::metadata_key("next_operation_id"); + +const DBRegisterKeyIntrospect kMetadataKeyIntrospect( + DBFormat::metadata_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kLastBreadcrumbKeyIntrospect( + kLastBreadcrumbKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +// TODO(peter): This function is duplicated in UIAppState.mm. +void DeleteOld(const string& old_dir) { + int files = 0; + int dirs = 0; + if (DirRemove(old_dir, true, &files, &dirs)) { + LOG("%s: removed %d files, %d dirs", old_dir, files, dirs); + } +} + +} // namespace + +const string AppState::kLinkEndpoint = "link"; +const string AppState::kLoginEndpoint = "login"; +const string AppState::kLoginResetEndpoint = "login_reset"; +const string AppState::kMergeTokenEndpoint = "merge_token"; +const string AppState::kRegisterEndpoint = "register"; +const string AppState::kVerifyEndpoint = "verify"; + +AppState::AppState(const string& base_dir, const string& server_host, + int server_port, bool production) + : server_protocol_((server_host == "localhost") ? "http" : "https"), + server_host_(server_host), + server_port_(server_port), + base_dir_(base_dir), + library_dir_(JoinPath(base_dir_, LibraryPath())), + database_dir_(JoinPath(library_dir_, "Database")), + photo_dir_(JoinPath(library_dir_, "Photos")), + server_photo_dir_(JoinPath(library_dir_, "ServerPhotos")), + auth_path_(JoinPath(library_dir_, "co.viewfinder.auth")), + production_(production), + cloud_storage_(false), + store_originals_(false), + no_password_(false), + refresh_completed_(false), + upload_logs_(false), + account_setup_(false), + last_login_timestamp_(0), + fake_logout_(false) { +#ifdef FAKE_LOGIN + fake_logout_ = true; +#endif // FAKE_LOGIN +} + +AppState::~AppState() { + // Delete the async state first. This will block until all of the running + // async operations have completed. + Kill(); +} + +bool AppState::Init(InitAction init_action) { + WallTimer timer; + if (!OpenDB(init_action == INIT_RESET)) { + return false; + } + InitDirs(); + + VLOG("init: db: %.03f ms", timer.Milliseconds()); + VLOG("init: db size: %.2f MB", db()->DiskUsage() / (1024.0 * 1024.0)); + timer.Restart(); + + analytics_.reset(new Analytics(production_)); + VLOG("init: analytics: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + activity_table_.reset(new ActivityTable(this)); + VLOG("init: activity table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + async_.reset(new AsyncState); + VLOG("init: async state: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + comment_table_.reset(new CommentTable(this)); + VLOG("init: comment table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + day_table_.reset(new DayTable(this, NewDayTableEnv())); + VLOG("init: day table (pre-init): %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + episode_table_.reset(new EpisodeTable(this)); + VLOG("init: episode table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + image_index_.reset(new ImageIndex); + VLOG("init: image index: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + people_rank_.reset(new PeopleRank(this)); + VLOG("init: people rank: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + photo_table_.reset(new PhotoTable(this)); + VLOG("init: photo table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + placemark_histogram_.reset(new PlacemarkHistogram(this)); + VLOG("init: placemark histogram: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + placemark_table_.reset(new PlacemarkTable(this)); + VLOG("init: placemark table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + photo_storage_.reset(new PhotoStorage(this)); + VLOG("init: photo storage: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + viewpoint_table_.reset(new ViewpointTable(this)); + VLOG("init: viewpoint table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + // IMPORTANT: Initialization order dependencies: + // * ContactManager and NetworkQueue depend on NotificationManager + notification_manager_.reset(new NotificationManager(this)); + VLOG("init: notification manager: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + contact_manager_.reset(new ContactManager(this)); + VLOG("init: contact manager: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + net_queue_.reset(new NetworkQueue(this)); + VLOG("init: network queue: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + return true; +} + +void AppState::RunMaintenance(InitAction init_action) { + WallTimer timer; + + ProgressUpdateBlock progress_update = [this](const string msg) { + maintenance_progress_.Run(msg); + }; + + // The init_maintenance flag indicates that maintenance should be run in a + // "silent" mode because the database was just reset. + const bool init_maintenance = db()->Exists(kInitMaintenanceKey); + if (init_maintenance) { + progress_update = NULL; + } + + // Run database migrations as necessary. + // NOTE: This MUST be done first to move any DB tables into a state + // that the rest of the code has been upgraded to expect. + const bool migrated = MaybeMigrate(progress_update); + VLOG("init: db migration: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + +#if defined(ENABLE_DB_STATS) + // Compute database statistics. + // + // NOTE(peter): This takes 1.5 secs on my client and proportionally longer + // for larger users like Chris. + DBStats stats(this); + stats.ComputeStats(); + VLOG("init: db statistics: %0.3f ms", timer.Milliseconds()); + timer.Restart(); +#endif // DEVELOPMENT + + // Consistency check of database tables. + const bool fscked = MaybeFSCK(init_action == INIT_FSCK, progress_update); + VLOG("init: fscked? %s: %0.3f ms", fscked ? "yes" : "no", + timer.Milliseconds()); + timer.Restart(); + + // Possibly unquarantine photos. + const bool unquarantined = + photo_table_->MaybeUnquarantinePhotos(progress_update); + VLOG("init: unquarantine: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + // Start day table initialization after all other tables are + // initialized, as it may quickly start refreshing. We always + // force a full refresh if any of the maintenance tasks resulted + // in DB modifications. + const bool reset = day_table_->Initialize( + !init_maintenance && (migrated || fscked || unquarantined)); + VLOG("init: day table: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + ProcessPhotoDuplicateQueue(); + VLOG("init: process duplicate queue: %0.3f ms", timer.Milliseconds()); + timer.Restart(); + + if (init_maintenance) { + db()->Delete(kInitMaintenanceKey); + } + + maintenance_done_.Run(!init_maintenance && reset); +} + +bool AppState::NeedDeviceIdReset() const { + DCHECK(!device_uuid_.empty()); +#if (TARGET_IPHONE_SIMULATOR) + return false; +#endif // (TARGET_IPHONE_SIMULATOR) + return auth_.device_uuid() != device_uuid_; +} + +void AppState::SetUserAndDeviceId(int64_t user_id, int64_t device_id) { + if (auth_.user_id() == user_id && auth_.device_id() == device_id) { + return; + } + // If we're updating the device id, also update the device_uuid. + if (auth_.device_id() != device_id && device_id != 0) { + DCHECK(!device_uuid_.empty()); + auth_.set_device_uuid(device_uuid_); + } + auth_.set_user_id(user_id); + auth_.set_device_id(device_id); + set_last_login_timestamp(WallTime_Now()); + WriteAuthMetadata(); + VLOG("setting user_id=%d device_id=%d", user_id, device_id); + async_->dispatch_main([this, user_id, device_id] { + settings_changed_.Run(false); + if (user_id == 0 && device_id == 0) { + // User is being logged out: force a maintenance run. + dispatch_after_low_priority(0.5, [this] { + maintenance_done_.Run(false); + }); + } + }); +} + +void AppState::SetAuthCookies( + const string& user_cookie, const string& xsrf_cookie) { + bool changed = false; + if (auth_.user_cookie() != user_cookie) { + auth_.set_user_cookie(user_cookie); + changed = true; + } + if (auth_.xsrf_cookie() != xsrf_cookie) { + auth_.set_xsrf_cookie(xsrf_cookie); + changed = true; + } + if (changed) { + WriteAuthMetadata(); + } +} + +void AppState::ClearAuthMetadata() { + auth_.Clear(); + WriteAuthMetadata(); +} + +int64_t AppState::NewLocalOperationId() { + MutexLock l(&next_op_id_mu_); + const int64_t id = next_op_id_++; + db_->Put(kNextOperationIdKey, next_op_id_); + return id; +} + +DBHandle AppState::NewDBTransaction() { + return db_->NewTransaction(); +} + +DBHandle AppState::NewDBSnapshot() { + return db_->NewSnapshot(); +} + +void AppState::set_server_host(const Slice& host) { + server_host_ = host.as_string(); + db_->Put(kServerHostKey, host); +} + +void AppState::set_last_breadcrumb(const Breadcrumb& b) { + if (!last_breadcrumb_.get()) { + last_breadcrumb_.reset(new Breadcrumb); + } + last_breadcrumb_->CopyFrom(b); + db_->PutProto(kLastBreadcrumbKey, *last_breadcrumb_); +} + +void AppState::set_cloud_storage(bool v) { + cloud_storage_ = v; + db_->Put(kCloudStorageKey, cloud_storage_); +} + +void AppState::set_store_originals(bool v) { + store_originals_ = v; + db_->Put(kStoreOriginalsKey, store_originals_); +} + +void AppState::set_no_password(bool v) { + no_password_ = v; + db_->Put(kNoPasswordKey, no_password_); + async_->dispatch_main([this] { + settings_changed_.Run(true); + }); +} + +void AppState::set_refresh_completed(bool v) { + refresh_completed_ = true; + db_->Put(kRefreshCompletedKey, refresh_completed_); +} + +void AppState::set_upload_logs(bool v) { + upload_logs_ = v; + db_->Put(kUploadLogsKey, upload_logs_); +} + +void AppState::set_last_login_timestamp(WallTime v) { + last_login_timestamp_ = v; + db_->Put(kLastLoginTimestampKey, last_login_timestamp_); +} + +void AppState::set_registration_version(RegistrationVersion v) { + registration_version_ = v; + db_->Put(kRegistrationVersionKey, static_cast(registration_version_)); +} + +void AppState::set_system_message(const SystemMessage& msg) { + if (msg.identifier() == system_message_.identifier()) { + return; + } + system_message_ = msg; + db_->PutProto(kSystemMessageKey, system_message_); + LOG("setting system_message=%s", system_message_); + async_->dispatch_main([this] { + system_message_changed_.Run(); + }); +} + +void AppState::clear_system_message() { + // Call set_system_message, we want to notify watchers when the message is cleared. + set_system_message(SystemMessage()); +} + +bool AppState::network_wifi() const { + return net_manager_->network_wifi(); +} + +void AppState::Kill() { + async_.reset(NULL); +} + +bool AppState::OpenDB(bool reset) { + DirCreate(library_dir_); + db_.reset(NewDB(database_dir_)); + + if (reset || kResetState) { + VLOG("%s: recreating database", db_->dir()); + Clean(library_dir_); + } + + for (int i = 0; i < 2; ++i) { + if (!db_->Open(kCacheSize)) { +#ifndef DEVELOPMENT + // In non-development builds, try to stumble ahead if the database is + // corrupt or cannot otherwise be opened. + if (i == 0) { + Clean(library_dir_); + continue; + } +#endif // DEVELOPMENT + return false; + } + + bool empty = true; + for (DB::PrefixIterator iter(db_, ""); + iter.Valid(); + iter.Next()) { + empty = false; + break; + } + + if (empty) { + // A new database, initialize the format major key. + if (!InitDB()) { + return false; + } + } else { + // An existing database, make sure the format major key matches. + string value; + if (!db_->Get(kFormatKey, &value) || + kFormatValue != value) { + VLOG("%s: major format mismatch, recreating database", db_->dir()); + db_->Close(); + Clean(library_dir_); + continue; + } + } + + const string old_dir(JoinPath(library_dir_, "old")); + dispatch_after_background(5, [old_dir] { DeleteOld(old_dir); }); + InitVars(); + return true; + } + return false; +} + +bool AppState::InitDB() { + DBHandle updates = NewDBTransaction(); + updates->Put(kFormatKey, kFormatValue); + updates->Put(kInitMaintenanceKey, ""); + if (!updates->Commit()) { + VLOG("%s: unable to initialize database format", db_->dir()); + db_->Close(); + return false; + } + return true; +} + +void AppState::InitDirs() { + // Ensure that the photos directory exists. + DirCreate(photo_dir_); + DirCreate(server_photo_dir_); +} + +void AppState::InitVars() { + // Load the auth metadata, falling back to initializing the auth metadata + // from the old database keys. + if (!ReadFileToProto(auth_path_, &auth_)) { + auth_.set_device_id(db_->Get(kDeviceIdKey, 0)); + auth_.set_user_id(db_->Get(kUserIdKey, 0)); + auth_.set_user_cookie(db_->Get(kUserCookieKey)); + auth_.set_xsrf_cookie(db_->Get(kXsrfCookieKey)); + WriteAuthMetadata(); + db_->Delete(kDeviceIdKey); + db_->Delete(kUserIdKey); + db_->Delete(kUserCookieKey); + db_->Delete(kXsrfCookieKey); + } + + // If the auth doesn't have the device's unique identifier, set it. + if (!auth_.has_device_uuid()) { + DCHECK(!device_uuid_.empty()); + auth_.set_device_uuid(device_uuid_); + WriteAuthMetadata(); + } + +#ifdef RESET_AUTH + auth_.Clear(); +#endif // RESET_AUTH + + last_breadcrumb_.reset(new Breadcrumb); + if (!db_->GetProto(kLastBreadcrumbKey, last_breadcrumb_.get())) { + last_breadcrumb_.reset(NULL); + } + + next_op_id_ = db_->Get(kNextOperationIdKey, 1); + server_host_ = db_->Get(kServerHostKey, server_host_); + // Cloud storage is not currently available. + //cloud_storage_ = db_->Get(kCloudStorageKey, false); + store_originals_ = db_->Get(kStoreOriginalsKey, false); + no_password_ = db_->Get(kNoPasswordKey, false); + refresh_completed_ = db_->Get(kRefreshCompletedKey, false); + upload_logs_ = db_->Get(kUploadLogsKey, true); + last_login_timestamp_ = db_->Get(kLastLoginTimestampKey, 0); + registration_version_ = static_cast( + db_->Get(kRegistrationVersionKey, REGISTRATION_GOOGLE_FACEBOOK)); + db_->GetProto(kSystemMessageKey, &system_message_); + + if (user_id() && !last_login_timestamp_) { + // The login must have predated our support of last_login_timestamp, + // so initialize it now. + set_last_login_timestamp(WallTime_Now()); + } + + VLOG("device_id=%d, user_id=%d", device_id(), user_id()); + VLOG("cloud_storage=%s", cloud_storage_ ? "on" : "off"); + VLOG("store_originals=%s", store_originals_ ? "on" : "off"); + VLOG("no_password=%s", no_password_ ? "true" : "false"); + VLOG("refresh_completed=%s", refresh_completed_ ? "on" : "off"); + VLOG("upload_logs=%s", upload_logs_ ? "on" : "off"); + VLOG("registration_version=%s", registration_version_); + VLOG("server_host=%s", server_host_); + VLOG("system_message=%s", system_message_); +} + +void AppState::Clean(const string& lib_dir) { + // Move the existing database and photo directories to a new directory + // which will be deleted asynchronously in the background. + const string old_dir(JoinPath(lib_dir, "old")); + DirCreate(old_dir); + const string unique_dir(JoinPath(old_dir, NewUUID())); + DirCreate(unique_dir); + + const string kSubdirs[] = { + "Database", + "Photos", + }; + for (int i = 0; i < ARRAYSIZE(kSubdirs); ++i) { + FileRename(JoinPath(lib_dir, kSubdirs[i]), + JoinPath(unique_dir, kSubdirs[i])); + } +} + +bool AppState::MaybeFSCK(bool force, ProgressUpdateBlock progress_update) { + DBHandle updates = NewDBTransaction(); + const bool repair = true; + bool fscked = false; + + // NOTE: these should be ordered from most self-contained (e.g. have + // the least references to other assets) to the least self-contained. + if (photo_table_->FSCK(force, progress_update, updates)) { + fscked = true; + } + if (episode_table_->FSCK(force, progress_update, updates)) { + fscked = true; + } + if (viewpoint_table_->FSCK(force, progress_update, updates)) { + fscked = true; + } + if (activity_table_->FSCK(force, progress_update, updates)) { + fscked = true; + } + + if (repair) { + updates->Commit(); + return fscked; + } + updates->Abandon(); + return false; +} + +void AppState::WriteAuthMetadata() { + // Note, include the auth metadata in backup so that it transfers to new + // devices. + CHECK(WriteProtoToFile(auth_path_, auth_, false /* exclude_from_backup */)); +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/AppState.h b/clients/shared/AppState.h new file mode 100644 index 0000000..43033d1 --- /dev/null +++ b/clients/shared/AppState.h @@ -0,0 +1,337 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "AuthMetadata.pb.h" +#import "Callback.h" +#import "DB.h" +#import "DBFormat.h" +#import "ScopedPtr.h" +#import "SystemMessage.pb.h" +#import "WallTime.h" + +class ActivityTable; +class Analytics; +class AsyncState; +class Breadcrumb; +class CommentTable; +class ContactManager; +class DayTable; +class DayTableEnv; +class EpisodeTable; +class GeocodeManager; +class ImageIndex; +class NetworkManager; +class NetworkQueue; +class NotificationManager; +class PeopleRank; +class PhotoStorage; +class PhotoTable; +class PlacemarkHistogram; +class PlacemarkTable; +class SubscriptionManager; +class ViewpointTable; + +typedef CallbackSet1 MaintenanceDone; +typedef CallbackSet1 MaintenanceProgress; +typedef CallbackSet1 SettingsChanged; +typedef Callback ProgressUpdateBlock; + +class AppState { + public: + static const string kLinkEndpoint; + static const string kLoginEndpoint; + static const string kLoginResetEndpoint; + static const string kMergeTokenEndpoint; + static const string kRegisterEndpoint; + static const string kVerifyEndpoint; + + enum InitAction { + INIT_NORMAL, + INIT_FSCK, + INIT_RESET, + }; + + // Protocol versions. + enum ProtocolVersion { + INITIAL_VERSION = 0, + ADD_HEADERS_VERSION = 1, + TEST_VERSION = 2, + RENAME_EVENT_VERSION = 3, + ADD_TO_VIEWPOINT_VERSION = 4, + QUERY_EPISODES_VERSION = 5, + UPDATE_POST_VERSION = 6, + UPDATE_SHARE_VERSION = 7, + ADD_OP_HEADER_VERSION = 8, + ADD_ACTIVITY_VERSION = 9, + EXTRACT_MD5_HASHES = 10, + INLINE_INVALIDATIONS = 11, + EXTRACT_FILE_SIZES = 12, + INLINE_COMMENTS = 13, + EXTRACT_ASSET_KEYS = 14, + SPLIT_NAMES = 15, + EXPLICIT_SHARE_ORDER = 16, + SUPPRESS_BLANK_COVER_PHOTO = 17, + SUPPORT_MULTIPLE_IDENTITIES_PER_CONTACT = 18, + RENAME_PHOTO_LABEL = 19, + SUPPRESS_AUTH_NAME = 20, + SEND_EMAIL_TOKEN = 21, + SUPPORT_REMOVED_FOLLOWERS = 22, + SUPPRESS_COPY_TIMESTAMP = 23, + SUPPORT_CONTACT_LIMITS = 24, + SUPPRESS_EMPTY_TITLE = 25, + }; + + static ProtocolVersion protocol_version() { return SUPPRESS_EMPTY_TITLE; } + + public: + AppState(const string& base_dir, const string& server_host, + int server_port, bool production); + virtual ~AppState(); + + virtual InitAction GetInitAction() = 0; + virtual bool Init(InitAction init_action); + virtual void RunMaintenance(InitAction init_action); + + // Setup commit trigger that adds an update callback looking for the + // specified "viewpoint_id". + virtual void SetupViewpointTransition(int64_t viewpoint_id, const DBHandle& updates) = 0; + + // Returns true if a new device id needs to be generated by the + // Viewfinder backend. This happens when the physical device + // changes (e.g. as on a backup/restore to a new device). + bool NeedDeviceIdReset() const; + + // Returns true if the user has enabled cloud storage and has a sufficient + // subscription to do so. + virtual bool CloudStorageEnabled() = 0; + + // Delete the specified asset. + virtual void DeleteAsset(const string& key) = 0; + + // Process the duplicate photo queue. + virtual void ProcessPhotoDuplicateQueue() = 0; + + // Generates the viewfinder images for the specified photo, invoking + // "completion" when done. + virtual void LoadViewfinderImages(int64_t photo_id, const DBHandle& db, + Callback completion) = 0; + + // Returns the seconds from GMT for the current time zone at the specified + // date. + virtual int TimeZoneOffset(WallTime t) const = 0; + + void SetUserAndDeviceId(int64_t user_id, int64_t device_id); + void SetDeviceId(int64_t v) { + SetUserAndDeviceId(auth_.user_id(), v); + } + void SetUserId(int64_t v) { + SetUserAndDeviceId(v, auth_.device_id()); + } + void SetAuthCookies(const string& user_cookie, const string& xsrf_cookie); + + // Clear user/device id and cookies. Used when logging out. + void ClearAuthMetadata(); + + // Provides a client-local monotonic sequence for operation + // ids. These should be stored with ServerOperation protobufs for + // use with JSON service requests. The local operation ids should be + // used to encode activity ids corresponding to each server + // operation. This allows a non-connected client to generate + // activities locally which will be linkable to server-side + // activities when eventual connectivity allows the operation to run + // and resultant notifications to be queried. + int64_t NewLocalOperationId(); + + // Creates a transactional database handle. Data written or deleted + // during the transaction will be visible when using the DBHandle + // for access. However, the data will not be visible from db() until + // the transaction is committed. No locking is provided by the + // underlying database, so the most recent write will always "win" + // in the case of concurrent write ops to the same key. + DBHandle NewDBTransaction(); + + // Creates a snapshot of the underlying database. The snapshot will be + // valid as long as a reference exists to this handle. No mutating + // database calls are allowed to the returned db handle. + DBHandle NewDBSnapshot(); + + virtual WallTime WallTime_Now() { return ::WallTime_Now(); } + + const DBHandle& db() const { return db_; } + + ActivityTable* activity_table() const { return activity_table_.get(); } + Analytics* analytics() { return analytics_.get(); } + AsyncState* async() { return async_.get(); } + CommentTable* comment_table() const { return comment_table_.get(); } + ContactManager* contact_manager() { return contact_manager_.get(); } + DayTable* day_table() const { return day_table_.get(); } + EpisodeTable* episode_table() const { return episode_table_.get(); } + GeocodeManager* geocode_manager() const { return geocode_manager_.get(); } + ImageIndex* image_index() const { return image_index_.get(); } + NetworkManager* net_manager() const { return net_manager_.get(); } + NetworkQueue* net_queue() const { return net_queue_.get(); } + NotificationManager* notification_manager() const { return notification_manager_.get(); } + PeopleRank* people_rank() const { return people_rank_.get(); } + PhotoStorage* photo_storage() const { return photo_storage_.get(); } + PhotoTable* photo_table() const { return photo_table_.get(); } + PlacemarkHistogram* placemark_histogram() const { return placemark_histogram_.get(); } + PlacemarkTable* placemark_table() const { return placemark_table_.get(); } + virtual SubscriptionManager* subscription_manager() const = 0; + ViewpointTable* viewpoint_table() const { return viewpoint_table_.get(); } + + const string& server_protocol() const { return server_protocol_; } + const string& server_host() const { return server_host_; } + void set_server_host(const Slice& host); + int server_port() const { return server_port_; } + + const string& photo_dir() const { return photo_dir_; } + const string& server_photo_dir() const { return server_photo_dir_; } + + bool is_registered() const { return !fake_logout_ && auth_.user_id(); } + int64_t device_id() const { return auth_.device_id(); } + int64_t user_id() const { return auth_.user_id(); } + string device_uuid() const { return device_uuid_; } + const AuthMetadata& auth() const { return auth_; } + + const Breadcrumb* last_breadcrumb() const { return last_breadcrumb_.get(); } + void set_last_breadcrumb(const Breadcrumb& b); + + // cloud_storage() is the value of the user's cloud storage preference. + // This preference should be ignored unless the user has a sufficient + // subscription; use CloudStorageEnabled() instead of accessing this + // value directly in most cases. + bool cloud_storage() const { return cloud_storage_; } + void set_cloud_storage(bool v); + + bool store_originals() const { return store_originals_; } + void set_store_originals(bool v); + + bool no_password() const { return no_password_; } + void set_no_password(bool v); + + // Returns true if at least one full refresh has completed since user authentication. + bool refresh_completed() { return refresh_completed_; } + void set_refresh_completed(bool v); + + bool upload_logs() const { return upload_logs_; } + void set_upload_logs(bool v); + + WallTime last_login_timestamp() const { return last_login_timestamp_; } + void set_last_login_timestamp(WallTime v); + + enum RegistrationVersion { + REGISTRATION_GOOGLE_FACEBOOK, + REGISTRATION_EMAIL, + }; + static RegistrationVersion current_registration_version() { + return REGISTRATION_EMAIL; + } + RegistrationVersion registration_version() const { return registration_version_; } + void set_registration_version(RegistrationVersion v); + + const SystemMessage& system_message() const { return system_message_; } + void clear_system_message(); + void set_system_message(const SystemMessage& msg); + + bool account_setup() const { return account_setup_; } + void set_account_setup(bool account_setup) { account_setup_ = account_setup; } + + MaintenanceDone* maintenance_done() { return &maintenance_done_; } + MaintenanceProgress* maintenance_progress() { return &maintenance_progress_; } + CallbackSet1* network_ready() { return &network_ready_; } + CallbackSet* app_did_become_active() { return &app_did_become_active_; } + CallbackSet* app_will_resign_active() { return &app_will_resign_active_; } + // Callbacks for settings changed or downloaded. + // Takes a bool as argument. This bool is true if run after + // settings were downloaded from the server, in which case they + // should be applied but not uploaded again. + SettingsChanged* settings_changed() { return &settings_changed_; } + CallbackSet* system_message_changed() { return &system_message_changed_; } + + const string& device_model() const { return device_model_; } + const string& device_name() const { return device_name_; } + const string& device_os() const { return device_os_; } + + const string& locale_language() const { return locale_language_; } + const string& locale_country() const { return locale_country_; } + const string& test_udid() const { return test_udid_; } + + virtual bool network_wifi() const; + virtual string timezone() const = 0; + + protected: + void Kill(); + bool OpenDB(bool reset); + bool InitDB(); + void InitDirs(); + virtual void InitVars(); + virtual DayTableEnv* NewDayTableEnv() = 0; + virtual void Clean(const string& dir); + virtual bool MaybeMigrate(ProgressUpdateBlock progress_update) = 0; + bool MaybeFSCK(bool force, ProgressUpdateBlock progress_update); + void WriteAuthMetadata(); + + protected: + const string server_protocol_; + string server_host_; + const int server_port_; + const string base_dir_; + const string library_dir_; + const string database_dir_; + const string photo_dir_; + const string server_photo_dir_; + const string auth_path_; + AuthMetadata auth_; + ScopedPtr last_breadcrumb_; + const bool production_; + string device_uuid_; + bool cloud_storage_; + bool store_originals_; + bool no_password_; + bool initial_contact_import_done_; + bool refresh_completed_; + bool upload_logs_; + bool account_setup_; + WallTime last_login_timestamp_; + RegistrationVersion registration_version_; + SystemMessage system_message_; + string device_model_; + string device_name_; + string device_os_; + string locale_language_; + string locale_country_; + string test_udid_; + MaintenanceDone maintenance_done_; + MaintenanceProgress maintenance_progress_; + CallbackSet1 network_ready_; + CallbackSet app_did_become_active_; + CallbackSet app_will_resign_active_; + SettingsChanged settings_changed_; + CallbackSet system_message_changed_; + DBHandle db_; + ScopedPtr activity_table_; + ScopedPtr analytics_; + ScopedPtr async_; + ScopedPtr comment_table_; + ScopedPtr contact_manager_; + ScopedPtr day_table_; + ScopedPtr episode_table_; + ScopedPtr geocode_manager_; + ScopedPtr image_index_; + ScopedPtr net_manager_; + ScopedPtr net_queue_; + ScopedPtr notification_manager_; + ScopedPtr people_rank_; + ScopedPtr photo_storage_; + ScopedPtr photo_table_; + ScopedPtr placemark_histogram_; + ScopedPtr placemark_table_; + ScopedPtr viewpoint_table_; + int64_t next_op_id_; + mutable Mutex next_op_id_mu_; + bool fake_logout_; +}; + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/AsyncState.cc b/clients/shared/AsyncState.cc new file mode 100644 index 0000000..32d68e4 --- /dev/null +++ b/clients/shared/AsyncState.cc @@ -0,0 +1,244 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "AsyncState.h" +#import "Utils.h" + +AsyncState::Impl::Impl() + : inflight_(0), + running_(0), + alive_(true) { +} + +AsyncState::AsyncBlock* AsyncState::Impl::QueueAsyncBlock( + const DispatchBlock& block, bool force) { + AsyncBlock* ab = new AsyncBlock(block); + if (Enter(ab, force)) { + return ab; + } + delete ab; + return NULL; +} + +bool AsyncState::Impl::Enter(AsyncBlock* ab, bool force) { + MutexLock l(&mu_); + if (!force && !alive_) { + return false; + } + ++inflight_; + if (ab) { + blocks_.insert(ab); + } + return true; +} + +void AsyncState::Impl::Kill() { + mu_.Lock(); + // Set alive to false to prevent new ops from entering the running state. + alive_ = false; + // Bump the inflight count to prevent deletion while we're waiting for the + // running ops to finish. + ++inflight_; + mu_.Wait([this] { + return running_ == 0; + }); + CHECK_EQ(0, running_); + --inflight_; + + // Loop over any inflight (but not running) blocks and clear the block + // variable in order to release any memory associated with the block. + for (BlockMap::iterator iter(blocks_.begin()); + iter != blocks_.end(); + ++iter) { + AsyncBlock* ab = *iter; + ab->clear(); + } + + const bool del = !inflight_; + mu_.Unlock(); + if (del) { + delete this; + } +} + +void AsyncState::Impl::Run(AsyncBlock* ab, bool force) { + dispatch_autoreleasepool([this, ab, force] { + if (Start(ab, force)) { + Finish(); + } + }); +} + +bool AsyncState::Impl::Start(AsyncBlock* ab, bool force) { + mu_.Lock(); + --inflight_; + const bool alive = force || alive_; + if (alive) { + ++running_; + } + if (ab) { + if (!alive) { + // If the AsyncState is being killed, clear the block before we release + // the lock in order to release any memory associated with the block. + ab->clear(); + } + blocks_.erase(ab); + } + const bool del = !alive && !inflight_ && !running_; + mu_.Unlock(); + if (del) { + delete this; + } + if (ab && ab->valid()) { + (*ab)(); + } + delete ab; + return alive; +} + +bool AsyncState::Impl::Finish() { + mu_.Lock(); + --running_; + const bool alive = alive_; + const bool del = !alive && !inflight_ && !running_; + mu_.Unlock(); + if (del) { + delete this; + } + return alive; +} + +void AsyncState::Impl::dispatch_main(const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_main([this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_main_async(const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_main_async([this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_network(bool force, const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block, force); + if (!ab) { + return; + } + ::dispatch_network([this, ab, force] { Run(ab, force); }); +} + +void AsyncState::Impl::dispatch_high_priority(const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_high_priority([this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_low_priority(const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_low_priority([this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_background(const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_background([this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_after_main( + double delay, const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_after_main(delay, [this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_after_network( + double delay, const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_after_network(delay, [this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_after_low_priority( + double delay, const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_after_low_priority(delay, [this, ab] { Run(ab); }); +} + +void AsyncState::Impl::dispatch_after_background( + double delay, const DispatchBlock& block) { + AsyncBlock* ab = QueueAsyncBlock(block); + if (!ab) { + return; + } + ::dispatch_after_background(delay, [this, ab] { Run(ab); }); +} + +AsyncState::AsyncState() + : impl_(new Impl), + parent_(NULL) { +} + +AsyncState::AsyncState(AsyncState* parent) + : impl_(new Impl), + parent_(parent) { + parent_->children_.insert(this); +} + +AsyncState::~AsyncState() { + Kill(); +} + +bool AsyncState::Enter() { + if (!impl_->Enter(NULL)) { + return false; + } + return impl_->Start(NULL); +} + +bool AsyncState::Exit() { + return impl_->Finish(); +} + +void AsyncState::Kill() { + ChildMap tmp_children; + tmp_children.swap(children_); + for (ChildMap::iterator iter(tmp_children.begin()); + iter != tmp_children.end(); + ++iter) { + AsyncState* child = *iter; + child->parent_ = NULL; + child->Kill(); + } + + if (parent_) { + parent_->children_.erase(this); + } + + if (impl_) { + impl_->Kill(); + impl_ = NULL; + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/AsyncState.h b/clients/shared/AsyncState.h new file mode 100644 index 0000000..dda2c7e --- /dev/null +++ b/clients/shared/AsyncState.h @@ -0,0 +1,196 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_ASYNC_STATE_H +#define VIEWFINDER_ASYNC_STATE_H + +#import +#import "Callback.h" +#import "Mutex.h" + +// The AsyncState class provides a mechanism for scheduling blocks to be run on +// the various dispatch threads with the added ability to "clean up" +// outstanding state synchronously when the AsyncState object is +// destroyed. AsyncState guarantees that when its destructor returns any +// scheduled blocks have either been run, or any reference to them has been +// released causing memory associated with the block to be released. Why is +// this important? Consider the following scenario: +// +// AsyncState* async = new AsyncState; +// PhotoHandle p = photo_table->LoadPhoto() +// async->dispatch_after_low_priority(5, []() { SomePhotoOperation(p); }); +// delete async; +// delete photo_table; +// +// If AsyncState held references to any scheduled (though un-run) blocks after +// its destructor returned the memory associated with those blocks could be +// released at an arbitrary point in the future. In the above example, the +// block would be deleted after photo_table was deleted causing the deleted +// PhotoHandle to be left referencing garbage memory. +class AsyncState { + typedef std::unordered_set ChildMap; + typedef Callback AsyncBlock; + + class Impl { + typedef std::unordered_set BlockMap; + + public: + Impl(); + + // Queue a new async block. If force is true, the operation is queued + // regardless of whether alive_ is false. Returns the newly queued + // AsyncBlock structure which should be passed to Run() or NULL if the + // operation could not be queued. + AsyncBlock* QueueAsyncBlock(const DispatchBlock& block, bool force = false); + + // Queue an async operation. If force is true, the operation is queued + // regardless of whether alive_ is false. + bool Enter(AsyncBlock* block, bool force = false); + + // Mark the async state as dead. + void Kill(); + + // Runs an async operation. + void Run(AsyncBlock* ab, bool force = false); + + // Start an async operation. The async state destructor blocks until all + // running operations complete. Returns true if the operation should be + // started and false if the async state is already in the process of being + // destroyed. + bool Start(AsyncBlock* ab, bool force = false); + + // Finish an async operation. Returns true if the async state is still + // alive and false if the state is in the process of being destroyed. + bool Finish(); + + // Runs the specified block on the main thread. If the current thread is + // the main thread, the block is run synchronously. + void dispatch_main(const DispatchBlock& block); + + // Runs the specified block on the main thread. The block is run + // asynchronously even if the current thread is the main thread. + void dispatch_main_async(const DispatchBlock& block); + + // Runs the specified block on the network thread. If force is true, the block + // is run regardless of whether the AsyncState object is being destroyed or + // not. + void dispatch_network(bool force, const DispatchBlock& block); + + // Runs the specified block on a background thread at high priority + void dispatch_high_priority(const DispatchBlock& block); + + // Runs the specified block on a background thread at low priority + void dispatch_low_priority(const DispatchBlock& block); + + // Runs the specified block on a background thread at background (lower than "low") priority. + void dispatch_background(const DispatchBlock& block); + + // Runs the specified block on the main/low-priority/background queue after + // the specified delay (in seconds) has elapsed. + void dispatch_after_main(double delay, const DispatchBlock& block); + void dispatch_after_network(double delay, const DispatchBlock& block); + void dispatch_after_low_priority(double delay, const DispatchBlock& block); + void dispatch_after_background(double delay, const DispatchBlock& block); + + private: + Mutex mu_; + BlockMap blocks_; + int inflight_; + int running_; + bool alive_; + }; + + public: + AsyncState(); + // Creates a new child async state that will be killed (but not deleted) when + // the parent async state is killed but can also be independently killed via + // an explicit delete before the parent is killed. In other words, the child + // async state has a lifetime that is limited by the lifetime of the parent. + // + // Usage: Async operations started by a class (e.g. PhotoStorage) need to be + // stopped before either the class is deleted or the associated AppState is + // deleted. PhotoStorage, in particular, presents a peculiar case because it + // is usually contained within an AppState and therefore presents no + // problem. But in tests a standalone PhotoStorage is used which is destroyed + // before the associated AppState. PhotoStorage creates a child AsyncState of + // AppState::async() so that its async operations are stopped if either the + // AppState is destroyed or the PhotoStorage is destroyed. + AsyncState(AsyncState* parent); + ~AsyncState(); + + // Called in conjunction with each other when there is not a + // discrete block of code to run. + bool Enter(); + // Returns false if the async state is being destroyed. + bool Exit(); + + // Runs the specified block on the main thread. If the current thread is the + // main thread, the block is run synchronously. + void dispatch_main(const DispatchBlock& block) { + impl_->dispatch_main(block); + } + + // Runs the specified block on the main thread. The block is run + // asynchronously even if the current thread is the main thread. + void dispatch_main_async(const DispatchBlock& block) { + impl_->dispatch_main_async(block); + } + + // Runs the specified block on the network queue thread. If force is true, the + // block is run regardless of whether the AsyncState object is being + // destroyed or not. + void dispatch_network(bool force, const DispatchBlock& block) { + impl_->dispatch_network(force, block); + } + + // The following three functions share a pool of background threads with a prioritized queue. + // (the priority in the function names refers to the priority in this queue, not thread scheduling + // priority). Guidance on priorities: + // * High: User-initiated operations. + // * Low: Background actions that may have user-visible results, like asset scans and day table refreshes. + // * Background: Mostly-invisible operations, such as garbage collection, or expensive operations like + // duplicate queue processing. + + // Run the specified block on a background thread at high priority. + void dispatch_high_priority(const DispatchBlock& block) { + impl_->dispatch_high_priority(block); + } + + // Run the specified block on a background thread at low priority. + void dispatch_low_priority(const DispatchBlock& block) { + impl_->dispatch_low_priority(block); + } + + // Run the specified block on a background thread at background (lower than "low") priority. + void dispatch_background(const DispatchBlock& block) { + impl_->dispatch_background(block); + } + + // Runs the specified block on the main/low-priority/background queue after + // the specified delay (in seconds) has elapsed. + void dispatch_after_network(double delay, const DispatchBlock& block) { + impl_->dispatch_after_network(delay, block); + } + + void dispatch_after_main(double delay, const DispatchBlock& block) { + impl_->dispatch_after_main(delay, block); + } + + void dispatch_after_low_priority(double delay, const DispatchBlock& block) { + impl_->dispatch_after_low_priority(delay, block); + } + + void dispatch_after_background(double delay, const DispatchBlock& block) { + impl_->dispatch_after_background(delay, block); + } + + protected: + void Kill(); + + protected: + Impl* impl_; + AsyncState* parent_; + ChildMap children_; +}; + +#endif // VIEWFINDER_ASYNC_STATE_H diff --git a/clients/shared/AuthMetadata.proto b/clients/shared/AuthMetadata.proto new file mode 100644 index 0000000..5e7a79c --- /dev/null +++ b/clients/shared/AuthMetadata.proto @@ -0,0 +1,18 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "AuthMetadataPB"; + +message AuthMetadata { + optional int64 user_id = 1; + optional int64 device_id = 2; + optional bytes user_cookie = 3; + optional bytes xsrf_cookie = 4; + // This value differs from "device_id" in that it is + // furnished by the device itself instead of by Viewfinder. + // It's used to determine whether the device has changed + // out from under the database, as in a backup / restore to + // a new physical device. + optional string device_uuid = 5; +} diff --git a/clients/shared/Breadcrumb.proto b/clients/shared/Breadcrumb.proto new file mode 100644 index 0000000..d0e1b29 --- /dev/null +++ b/clients/shared/Breadcrumb.proto @@ -0,0 +1,18 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +import "Location.proto"; +import "Placemark.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "BreadcrumbPB"; + +message Breadcrumb { + optional Location location = 1; + optional Placemark placemark = 2; + // The time when the breadcrumb was gathered. + optional double timestamp = 3; + // Additional non-programmatic data about the breadcrumb (e.g. app state when + // the breadcrumb was gathered). + optional string debug = 4; +} diff --git a/clients/shared/Callback.h b/clients/shared/Callback.h new file mode 100644 index 0000000..3dcf9b8 --- /dev/null +++ b/clients/shared/Callback.h @@ -0,0 +1,228 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +// ==================== PAY ATTENTION, THIS IS IMPORTANT ==================== +// +// Combining ARC (automatic reference counting) and non-ARC code (e.g. plain +// C++) with templates is dangerous due to easy violations of the +// one-definition-rule +// (http://en.wikipedia.org/wiki/One_Definition_Rule). Consider +// Callback::Release(). When compiled with ARC it will yield one definition and +// when compiled without ARC it will yield another. At link time, the compiler +// will arbitrarily choose one definition which might differ from the +// definition it chose for Callback::Acquire() leading to a memory leak or +// memory corruption. +// +// The current solution is fragile: Ensure that specializations of Callback<> +// and CallbackSetBase<> are either always instatiated with ARC enabled or +// always with ARC disabled. This is accomplished by suppressing instantiation +// of the specializations of Callback<> and CallbackSetBase<> used by C++ code +// and then explicitly instantiating them in C++ code. See the suppressions at +// the end of this file and the instantiations in Callback.cc. Fragile. +// +// ========================================================================== + +#ifndef VIEWFINDER_CALLBACK_H +#define VIEWFINDER_CALLBACK_H + +#import +#import +#import "Mutex.h" +#import "STLUtils.h" + +#if __has_extension(blocks) && !defined(__OBJC__) +#error "Objc++ compilation required" +#endif // __has_extension(blocks) && !defined(__OBJC__) + +// Callback is like std::function<> except that it knows about blocks and +// properly copies and releases them. +template class Callback; + +template +class Callback { + typedef std::function FunctionType; + +#if __has_extension(blocks) + typedef R (^BlockType)(ArgTypes...); +#endif // __has_extension(blocks) + + public: + Callback() { + } + // Needed so that we can pass NULL (a.k.a. 0) to the Callback constructor. + Callback(int) { + } + Callback(std::nullptr_t) { + } + Callback(const FunctionType& f) + : func_(f) { + Acquire(&func_); + } + Callback(const Callback& f) + : Callback(f.func_) { + } +#if __has_extension(blocks) + Callback(BlockType b) + : func_((BlockType)[b copy]) { + } +#endif // __has_extension(blocks) + template + Callback(F f) + : func_(f) { + Acquire(&func_); + } + ~Callback() { + clear(); + } + + R operator()(ArgTypes... args) const { + return func_(args...); + } + + void clear() { + Release(&func_); + } + + void swap(Callback* other) { + func_.swap(other->func_); + } + + bool valid() const { + // Need the ternary operator in order to get the bool operator invoked. + return func_ ? true : false; + } + explicit operator bool() const { + return valid(); + } + + private: + static void Acquire(FunctionType* f) { +#if __has_extension(blocks) + const BlockType* block = f->template target(); + if (block) { + *f = (BlockType)[*block copy]; + } +#endif // __has_extension(blocks) + } + + static void Release(FunctionType* f) { +#if __has_extension(blocks) + // Nothing to do with objc. ARC will take care of releasing the block. +#endif // __has_extension(blocks) + *f = nullptr; + } + + private: + FunctionType func_; +}; + +template +class CallbackSetBase { + typedef std::function FunctionType; + typedef Callback CallbackType; + typedef std::unordered_map CallbackMap; + typedef std::set RemoveSet; + + public: + CallbackSetBase() + : next_id_(1) { + } + ~CallbackSetBase() { + Clear(); + } + + // Adds the given callback to the set. Returns an id (a nonzero + // integer) which can be used to Remove() the callback later. See + // also AddSingleShot (defined in subclasses below). + int Add(CallbackType callback) { + MutexLock l(&mu_); + const int id = next_id_++; + callbacks_[id].swap(&callback); + return id; + } + + void AddSingleShot(const CallbackType& callback) { + MutexLock l(&mu_); + const int id = next_id_++; + callbacks_[id] = [callback, id, this](ArgTypes&&... args) { + callback(args...); + Remove(id); + }; + } + + void Remove(int id) { + if (mu_.TryLock()) { + callbacks_.erase(id); + mu_.Unlock(); + } else { + MutexLock l(&removed_mu_); + removed_.insert(id); + } + } + + void Run(ArgTypes... args) { + MutexLock l(&mu_); + ApplyRemoved(); + for (typename CallbackMap::const_iterator iter(callbacks_.begin()); + iter != callbacks_.end(); + ++iter) { + const CallbackType& callback = iter->second; + callback(args...); + } + ApplyRemoved(); + } + + void Clear() { + MutexLock l(&mu_); + ::Clear(&callbacks_); + } + + void Swap(CallbackSetBase* other) { + MutexLock l(&mu_); + MutexLock l2(&other->mu_); + ApplyRemoved(); + other->ApplyRemoved(); + std::swap(next_id_, other->next_id_); + callbacks_.swap(other->callbacks_); + } + + int size() const { + MutexLock l(&mu_); + return callbacks_.size(); + } + bool empty() const { + MutexLock l(&mu_); + return callbacks_.empty(); + } + + private: + void ApplyRemoved() { + MutexLock l(&removed_mu_); + for (RemoveSet::iterator iter(removed_.begin()); + iter != removed_.end(); + ++iter) { + callbacks_.erase(*iter); + } + removed_.clear(); + } + + private: + mutable Mutex mu_; + int next_id_; + CallbackMap callbacks_; + Mutex removed_mu_; + RemoveSet removed_; +}; + +using CallbackSet = CallbackSetBase<>; + +template +using CallbackSet1 = CallbackSetBase; + +template +using CallbackSet2 = CallbackSetBase; + +template +using CallbackSet3 = CallbackSetBase; + +#endif // VIEWFINDER_CALLBACK_H diff --git a/clients/shared/CommentMetadata.proto b/clients/shared/CommentMetadata.proto new file mode 100644 index 0000000..2423e9e --- /dev/null +++ b/clients/shared/CommentMetadata.proto @@ -0,0 +1,17 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +import "ContentIds.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "CommentMetadataPB"; + +message CommentMetadata { + optional CommentId comment_id = 1; + optional ViewpointId viewpoint_id = 2; + optional int64 user_id = 3; + optional string asset_id = 4; + optional double timestamp = 5; + optional string message = 6; + repeated string indexed_terms = 7; +} diff --git a/clients/shared/CommentTable.cc b/clients/shared/CommentTable.cc new file mode 100644 index 0000000..3ee5f12 --- /dev/null +++ b/clients/shared/CommentTable.cc @@ -0,0 +1,125 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import +#import "ActivityTable.h" +#import "AppState.h" +#import "CommentTable.h" +#import "DayTable.h" +#import "FullTextIndex.h" +#import "LazyStaticPtr.h" +#import "StringUtils.h" + +namespace { + +const DBRegisterKeyIntrospect kCommentKeyIntrospect( + DBFormat::comment_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kCommentServerKeyIntrospect( + DBFormat::comment_server_key(), NULL, [](Slice value) { + return value.ToString(); + }); + +LazyStaticPtr kDocIDRE = { "([0-9]+),([0-9]+)" }; + +const string kCommentIndexName = "com"; + +} // namespace + +//// +// Comment + +CommentTable_Comment::CommentTable_Comment( + AppState* state, const DBHandle& db, int64_t id) + : state_(state), + db_(db) { + mutable_comment_id()->set_local_id(id); +} + +void CommentTable_Comment::MergeFrom(const CommentMetadata& m) { + // Some assertions that immutable properties don't change. + if (viewpoint_id().has_server_id() && m.viewpoint_id().has_server_id()) { + DCHECK_EQ(viewpoint_id().server_id(), m.viewpoint_id().server_id()); + } + if (has_user_id() && m.has_user_id()) { + DCHECK_EQ(user_id(), m.user_id()); + } + if (has_timestamp() && m.has_timestamp()) { + DCHECK_LT(fabs(timestamp() - m.timestamp()), 0.000001); + } + + CommentMetadata::MergeFrom(m); +} + +void CommentTable_Comment::MergeFrom(const ::google::protobuf::Message&) { + DIE("MergeFrom(Message&) should not be used"); +} + +void CommentTable_Comment::SaveHook(const DBHandle& updates) { + // Invalidate the activity which posted this comment, so that any + // saved changes are updated in the relevant conversation. + if (comment_id().has_server_id()) { + ActivityHandle ah = state_->activity_table()->GetCommentActivity( + comment_id().server_id(), updates); + if (ah.get()) { + state_->day_table()->InvalidateActivity(ah, updates); + } + } +} + +//// +// CommentTable + +CommentTable::CommentTable(AppState* state) + : ContentTable( + state, DBFormat::comment_key(), DBFormat::comment_server_key()), + comment_index_(new FullTextIndex(state_, kCommentIndexName)) { +} + +CommentTable::~CommentTable() { +} + +CommentHandle CommentTable::LoadComment(const CommentId& id, const DBHandle& db) { + CommentHandle ch; + if (id.has_local_id()) { + ch = LoadComment(id.local_id(), db); + } + if (!ch.get() && id.has_server_id()) { + ch = LoadComment(id.server_id(), db); + } + return ch; +} + +void CommentTable::SaveContentHook(Comment* comment, const DBHandle& updates) { + vector terms; + comment_index_->ParseIndexTerms(0, comment->message(), &terms); + // Inline the viewpoint id into our "docid" so we can use this index to find viewpoints + // without extra database lookups. + const string docid(Format("%d,%d", comment->viewpoint_id().local_id(), comment->comment_id().local_id())); + comment_index_->UpdateIndex(terms, docid, FullTextIndex::TimestampSortKey(comment->timestamp()), + comment->mutable_indexed_terms(), updates); +} + +void CommentTable::DeleteContentHook(Comment* comment, const DBHandle& updates) { + comment_index_->RemoveTerms(comment->mutable_indexed_terms(), updates); +} + +void CommentTable::Search(const Slice& query, CommentSearchResults* results) { + ScopedPtr parsed_query(FullTextQuery::Parse(query)); + for (ScopedPtr iter(comment_index_->Search(state_->db(), *parsed_query)); + iter->Valid(); + iter->Next()) { + const Slice docid = iter->doc_id(); + Slice viewpoint_id_slice, comment_id_slice; + CHECK(RE2::FullMatch(docid, *kDocIDRE, &viewpoint_id_slice, &comment_id_slice)); + const int64_t viewpoint_id = FastParseInt64(viewpoint_id_slice); + const int64_t comment_id = FastParseInt64(comment_id_slice); + results->push_back(std::make_pair(viewpoint_id, comment_id)); + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/CommentTable.h b/clients/shared/CommentTable.h new file mode 100644 index 0000000..8de3fba --- /dev/null +++ b/clients/shared/CommentTable.h @@ -0,0 +1,75 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_COMMENT_TABLE_H +#define VIEWFINDER_COMMENT_TABLE_H + +#import "CommentMetadata.pb.h" +#import "ContentTable.h" +#import "WallTime.h" + +class FullTextIndex; + +// The CommentTable class maintains the mappings: +// -> +// -> + +class CommentTable_Comment : public CommentMetadata { + public: + virtual void MergeFrom(const CommentMetadata& m); + // Unimplemented; exists to get the compiler not to complain about hiding the base class's overloaded MergeFrom. + virtual void MergeFrom(const ::google::protobuf::Message&); + + protected: + bool Load() { return true; } + void SaveHook(const DBHandle& updates); + void DeleteHook(const DBHandle& updates) {} + + int64_t local_id() const { return comment_id().local_id(); } + const string& server_id() const { return comment_id().server_id(); } + + CommentTable_Comment(AppState* state, const DBHandle& db, int64_t id); + + protected: + AppState* state_; + DBHandle db_; +}; + +class CommentTable : public ContentTable { + typedef CommentTable_Comment Comment; + + public: + CommentTable(AppState* state); + virtual ~CommentTable(); + + ContentHandle NewComment(const DBHandle& updates) { + return NewContent(updates); + } + ContentHandle LoadComment(int64_t id, const DBHandle& db) { + return LoadContent(id, db); + } + ContentHandle LoadComment(const string& server_id, const DBHandle& db) { + return LoadContent(server_id, db); + } + ContentHandle LoadComment(const CommentId& id, const DBHandle& db); + + void SaveContentHook(Comment* comment, const DBHandle& updates); + void DeleteContentHook(Comment* comment, const DBHandle& updates); + + // List of (viewpoint_id, comment_id) pairs. + typedef vector > CommentSearchResults; + void Search(const Slice& query, CommentSearchResults* results); + + FullTextIndex* comment_index() const { return comment_index_.get(); } + + private: + ScopedPtr comment_index_; +}; + +typedef CommentTable::ContentHandle CommentHandle; + +#endif // VIEWFINDER_COMMENT_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Compat.android.cc b/clients/shared/Compat.android.cc new file mode 100644 index 0000000..b8c361e --- /dev/null +++ b/clients/shared/Compat.android.cc @@ -0,0 +1,16 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault. + +#import +#import +#import "Compat.android.h" + +time_t timegm(struct tm* const t) { + // time_t is signed on Android. + static const time_t kTimeMax = ~(1 << (sizeof(time_t) * CHAR_BIT - 1)); + static const time_t kTimeMin = (1 << (sizeof(time_t) * CHAR_BIT - 1)); + time64_t result = timegm64(t); + if (result < kTimeMin || result > kTimeMax) + return -1; + return result; +} diff --git a/clients/shared/Compat.android.h b/clients/shared/Compat.android.h new file mode 100644 index 0000000..31099f2 --- /dev/null +++ b/clients/shared/Compat.android.h @@ -0,0 +1,22 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault. +// +// Android-specific implementations. + +#ifdef OS_ANDROID + +#ifndef VIEWFINDER_COMPAT_ANDROID_H +#define VIEWFINDER_COMPAT_ANDROID_H + +#import +#import +#import + +// Source: http://src.chromium.org/svn/trunk/src/base/os_compat_android.cc +// Android has only timegm64() and no timegm(). +// We replicate the behaviour of timegm() when the result overflows time_t. +time_t timegm(struct tm* const t); + +#endif // VIEWFINDER_COMPAT_ANDROID_H + +#endif // OS_ANDROID diff --git a/clients/shared/ContactManager.cc b/clients/shared/ContactManager.cc new file mode 100644 index 0000000..ae336d0 --- /dev/null +++ b/clients/shared/ContactManager.cc @@ -0,0 +1,2307 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import "Analytics.h" +#import "AppState.h" +#import "AsyncState.h" +#import "ContactManager.h" +#import "ContactMetadata.pb.h" +#import "DB.h" +#import "DigestUtils.h" +#import "FullTextIndex.h" +#import "IdentityManager.h" +#import "InvalidateMetadata.pb.h" +#import "LazyStaticPtr.h" +#import "LocaleUtils.h" +#import "Logging.h" +#import "NetworkManager.h" +#import "NotificationManager.h" +#import "PeopleRank.h" +#import "PhoneUtils.h" +#import "STLUtils.h" +#import "StringUtils.h" +#import "Timer.h" + +const string ContactManager::kContactSourceGmail = "gm"; +const string ContactManager::kContactSourceFacebook = "fb"; +const string ContactManager::kContactSourceIOSAddressBook = "ip"; +const string ContactManager::kContactSourceManual = "m"; + +const string ContactManager::kContactIndexName = "con"; +const string ContactManager::kUserIndexName = "usr"; + +const string ContactManager::kUserTokenPrefix = "_user"; + +namespace { + +const string kContactSelectionKey = DBFormat::metadata_key("contact_selection"); +// The contacts_format key is no longer used but may exist in old databases. +// Leaving it commented out here as a reminder to not re-use the name. +// const string kFormatKey = DBFormat::metadata_key("contacts_format"); +const string kMergeAccountsCompletionKey = + DBFormat::metadata_key("merge_accounts_completion_key"); +const string kMergeAccountsOpIdKey = + DBFormat::metadata_key("merge_accounts_op_id_key"); +const string kQueuedUpdateSelfKey = DBFormat::metadata_key("queued_update_self"); + +const string kNewUserCallbackTriggerKey = "ContactManagerNewUsers"; + +const int kUploadContactsLimit = 50; + +// Parse everything between unicode separator characters. This +// will include all punctuation, both internal to the string and +// leading and trailing. +LazyStaticPtr kWordUnicodeRE = { "([^\\pZ]+)" }; +LazyStaticPtr kWhitespaceUnicodeRE = { "([\\pZ]+)" }; + +LazyStaticPtr kIdRE = { "u/([[:digit:]]+)" }; +LazyStaticPtr kQueueRE = { "cq/([[:digit:]]+)" }; +LazyStaticPtr kUpdateQueueRE = { "cuq/([[:digit:]]+)" }; +LazyStaticPtr kContactUploadQueueRE = { "ccuq/(.*)" }; +LazyStaticPtr kContactRemoveQueueRE = { "ccrq/(.*)" }; +LazyStaticPtr kNewUserRE = { "nu/([[:digit:]]+)" }; + +// Format used to build filter regexp (case-insensitve match) on the filter +// string or on the filter string alone or with a leading separator character. +const char* kFilterREFormat = "(?i)(?:^|[\\s]|[[:punct:]])(%s)"; + +// kEmailFullRE is used to decide when to start a ResolveContact +// operation. We require at least one dot in the domain portion, and +// at least two characters after the last dot. +LazyStaticPtr kEmailFullRE = { "^[^ @]+@[^ @]+\\.[^. @]{2,}" }; + +LazyStaticPtr kUserTokenRE = { "_user([0-9]+)_" }; + +const DBRegisterKeyIntrospect kContactKeyIntrospect( + DBFormat::contact_key(""), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kDeprecatedContactIdKeyIntrospect( + DBFormat::deprecated_contact_id_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kContactSelectionKeyIntrospect( + kContactSelectionKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kContactSourceKeyIntrospect( + DBFormat::contact_source_key(""), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kUserIdKeyIntrospect( + DBFormat::user_id_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kUserIdentityKeyIntrospect( + DBFormat::user_identity_key(""), NULL, [](Slice value) { + return ToString(value); + }); + + +// Identities with higher priority should be given preference when choosing +// the primary identity. +int PrimaryIdentityPriority(const Slice& identity) { + if (IdentityManager::IsEmailIdentity(identity)) { + return 3; + } else if (IdentityManager::IsPhoneIdentity(identity)) { + return 2; + } else { + // Facebook, etc. + return 1; + } +} + +struct ContactMatchRankLess { + AppState* state; + WallTime now; + + ContactMatchRankLess(AppState* s) + : state(s), + now(state->WallTime_Now()) { + } + + bool operator()(const ContactManager::ContactMatch* a, const ContactManager::ContactMatch* b) const { + // TODO(pmattis): This sorts identities with viewfinder user ids to the + // top. Not clear if this is the right thing to do long term. + if (a->metadata.has_user_id() != b->metadata.has_user_id()) { + return !a->metadata.has_user_id() < !b->metadata.has_user_id(); + } else { + // This sorts identities which are registered before ones that + // are still prospective. + if (a->metadata.label_registered() != b->metadata.label_registered()) { + return !a->metadata.label_registered() < !b->metadata.label_registered(); + } + } + // Contacts with non-viewfinder identities came from the user's own contact import, + // so rank them above contacts that are only known because of transitive viewfinder + // connections. + if (IdentityManager::IsViewfinderIdentity(a->metadata.primary_identity()) != + IdentityManager::IsViewfinderIdentity(b->metadata.primary_identity())) { + return IdentityManager::IsViewfinderIdentity(a->metadata.primary_identity()) < + IdentityManager::IsViewfinderIdentity(b->metadata.primary_identity()); + } + // Ranks are considered in order of email, phone & facebook. + const int a_priority = PrimaryIdentityPriority(a->metadata.primary_identity()); + const int b_priority = PrimaryIdentityPriority(b->metadata.primary_identity()); + if (a_priority != b_priority) { + return a_priority > b_priority; + } + // Use people rank weight (sort descending) if user ids are available. + if (a->metadata.has_user_id() && b->metadata.has_user_id()) { + const double a_weight = state->people_rank()->UserRank(a->metadata.user_id(), now); + const double b_weight = state->people_rank()->UserRank(b->metadata.user_id(), now); + if (a_weight != b_weight) { + return a_weight > b_weight; + } + } + if (a->metadata.has_rank() != b->metadata.has_rank()) { + return !a->metadata.has_rank() < !b->metadata.has_rank(); + } + // If identity type is the same, use rank for direct comparison. + if (a->metadata.rank() != b->metadata.rank()) { + return a->metadata.rank() < b->metadata.rank(); + } + if (a->metadata.name() != b->metadata.name()) { + return a->metadata.name() < b->metadata.name(); + } + return a->metadata.primary_identity() < b->metadata.primary_identity(); + } +}; + +struct ContactMatchNameLess { + bool operator()(ContactManager::ContactMatch* a, ContactManager::ContactMatch* b) const { + // This method inlines parts of ContactNameLessThan so it can cache the sort keys. + if (!a->sort_key_initialized) { + a->sort_key_initialized = true; + a->sort_key = ContactManager::ContactNameForSort(a->metadata); + } + if (!b->sort_key_initialized) { + b->sort_key_initialized = true; + b->sort_key = ContactManager::ContactNameForSort(b->metadata); + } + if (a->sort_key != b->sort_key) { + return ContactManager::NameLessThan(a->sort_key, b->sort_key); + } + return a->metadata.primary_identity() < b->metadata.primary_identity(); + } +}; + +// Returns true if the given user can be shown as a user (rather than a contact) in search results. +bool IsViewfinderUser(const ContactMetadata& m) { + if (!m.has_user_id() || + m.has_merged_with() || + m.label_terminated() || + m.label_system()) { + // Non-users, merged accounts, terminated accounts, and system users are never shown. + return false; + } + if (m.label_friend()) { + // For friends, we receive terminate and merge notifications so the above data should always be + // accurate. Unregistered friends (i.e. prospective users in one of your conversations) will be + // shown as invited users. + return true; + } else { + // For non-friends, our information comes because we have their linked identity as a contact. + // If the user is unlinked from that identity, we will never find out whether the user is + // terminated, so we must not show it. + if (m.identities_size() == 0) { + return false; + } + + // Non-friends should not be shown as users unless they are fully registered. + return m.label_registered(); + } +} + +void HashString(SHA256_CTX* ctx, const Slice& s) { + SHA256_Update(ctx, s.data(), s.size()); + const char delimiter = 0; + SHA256_Update(ctx, &delimiter, 1); +} + +string ComputeContactId(const ContactMetadata& metadata) { + SHA256_CTX ctx; + SHA256_Init(&ctx); + HashString(&ctx, metadata.primary_identity()); + for (int i = 0; i < metadata.identities_size(); i++) { + if (metadata.identities(i).identity().empty()) { + continue; + } + HashString(&ctx, metadata.identities(i).identity()); + HashString(&ctx, metadata.identities(i).description()); + } + HashString(&ctx, "\0"); // Mark the end of the list + HashString(&ctx, metadata.name()); + HashString(&ctx, metadata.first_name()); + HashString(&ctx, metadata.last_name()); + HashString(&ctx, metadata.nickname()); + int64_t rank = metadata.rank(); + SHA256_Update(&ctx, &rank, sizeof(rank)); + uint8_t digest[SHA256_DIGEST_LENGTH]; + SHA256_Final(&ctx, digest); + // 256 bits is far more than we need for this case (and its size + // affects the fulltext index size, so truncate to 128 bits (22 + // bytes after b64 without padding). + const string contact_id = Format("%s:%s", metadata.contact_source(), + Base64HexEncode(Slice((const char*)digest, 16), false)); + if (metadata.has_contact_id()) { + CHECK_EQ(contact_id, metadata.contact_id()); + } + return contact_id; +} + +// Adds the given identity to the contact, updating primary_identity if necessary. +void AddIdentity(ContactMetadata* metadata, const string& identity) { + metadata->add_identities()->set_identity(identity); + if (metadata->primary_identity().empty() || + (PrimaryIdentityPriority(identity) > PrimaryIdentityPriority(metadata->primary_identity()))) { + metadata->set_primary_identity(identity); + } +} + +// Adds the given identity to the contact (which must be a user) and updates the identity-to-user index. +void AddIdentityAndSave(ContactMetadata* metadata, const string& identity, const DBHandle& updates) { + AddIdentity(metadata, identity); + updates->Put(DBFormat::user_identity_key(identity), ToString(metadata->user_id())); +} + +// Copies any identities from source to target if they are not already present. +// Does not update indexes so should only be used for transient objects. +void MergeIdentities(const ContactMetadata& source, ContactMetadata* target) { + StringSet identities; + for (int i = 0; i < target->identities_size(); i++) { + identities.insert(target->identities(i).identity()); + } + if (!source.primary_identity().empty() && + !ContainsKey(identities, source.primary_identity())) { + AddIdentity(target, source.primary_identity()); + identities.insert(source.primary_identity()); + } + for (int i = 0; i < source.identities_size(); i++) { + if (!ContainsKey(identities, source.identities(i).identity())) { + AddIdentity(target, source.identities(i).identity()); + } + } +} + +// Returns true if the two objects have at least one identity in common. +bool IdentitiesOverlap(const ContactMetadata& a, const ContactMetadata& b) { + StringSet identities; + if (!a.primary_identity().empty()) { + identities.insert(a.primary_identity()); + } + for (int i = 0; i < a.identities_size(); i++) { + identities.insert(a.identities(i).identity()); + } + if (!b.primary_identity().empty()) { + if (ContainsKey(identities, b.primary_identity())) { + return true; + } + } + for (int i = 0; i < b.identities_size(); i++) { + if (ContainsKey(identities, b.identities(i).identity())) { + return true; + } + } + return false; +} + +bool IsUploadableContactSource(const Slice& contact_source) { + return (contact_source == ContactManager::kContactSourceIOSAddressBook || + contact_source == ContactManager::kContactSourceManual); +} + +bool DecodeNewUserKey(Slice key, int64_t* user_id) { + return RE2::FullMatch(key, *kNewUserRE, user_id); +} + +} // namespace + +bool DecodeUserIdKey(Slice key, int64_t* user_id) { + return RE2::FullMatch(key, *kIdRE, user_id); +} + +bool IsValidEmailAddress(const Slice& address, string* error) { + vector parts = SplitAllowEmpty(address.ToString(), "@"); + if (parts.size() <= 1) { + *error = "You seem to be missing an \"@\" symbol."; + return false; + } + if (parts.size() > 2) { + *error = Format("I found %d \"@\" symbols. That's %d too many.", + parts.size() - 1, parts.size() - 2); + return false; + } + if (RE2::FullMatch(address, ".*[\\pZ\\pC].*")) { + *error = "There's a space in your email address - please remove it."; + return false; + } + const Slice user(parts[0]); + if (user.empty()) { + *error = "This email is missing a username (you know, the bit before the \"@\")."; + return false; + } + const Slice domain(parts[1]); + if (domain.empty()) { + *error = "This email is missing a domain (you know, the bit after the \"@\")."; + return false; + } + parts = SplitAllowEmpty(parts[1], "."); + if (parts.size() <= 1) { + *error = "This email is missing a domain (maybe it's .com? .edu?)."; + return false; + } + for (int i = 0; i < parts.size(); ++i) { + if (parts[i].empty()) { + if (i == parts.size() - 1) { + *error = "This email is missing a domain (maybe it's .com? .edu?)."; + } else { + *error = "I'm not sure what to do with \"..\"."; + } + return false; + } + } + if (address.size() > 1000) { + // Officially the limit is 254 characters; allow some extra room for non-compliant servers, unicode, etc. + // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address + *error = "That's too long to be an email address."; + return false; + } + return true; +} + +ContactManager::ContactManager(AppState* state) + : state_(state), + count_(0), + viewfinder_count_(0), + queued_update_self_(false), + queued_update_friend_(0), + user_index_(new FullTextIndex(state_, kUserIndexName)), + contact_index_(new FullTextIndex(state_, kContactIndexName)) { + + DBHandle updates = state_->NewDBTransaction(); + + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ++count_; + } + + // Count contacts with viewfinder user ids. + for (DB::PrefixIterator iter(updates, DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + LOG("contacts: unable to parse contact metadata: %s", iter.key()); + continue; + } + if (m.user_id() != state_->user_id() && IsViewfinderUser(m)) { + ++viewfinder_count_; + } + } + + LOG("contacts: %d contact%s, %d VF", + count_, Pluralize(count_), viewfinder_count_); + + queued_update_self_ = state_->db()->Get(kQueuedUpdateSelfKey, false); + QueueUpdateFriend(0); + + // Re-initialize any merge accounts operation watch. + ProcessMergeAccounts(updates->Get(kMergeAccountsOpIdKey), + updates->Get(kMergeAccountsCompletionKey), + NULL); + + state_->network_ready()->Add([this](int priority) { + MaybeQueueUploadContacts(); + MaybeQueueRemoveContacts(); + }); + + // Set up callbacks for handling notification mgr callbacks. + state_->notification_manager()->process_notifications()->Add( + [this](const QueryNotificationsResponse& p, const DBHandle& updates) { + ProcessQueryNotifications(p, updates); + }); + state_->notification_manager()->nuclear_invalidations()->Add( + [this](const DBHandle& updates) { + InvalidateAll(updates); + }); + + // We do not want to send the settings to the server if the + // "settings changed" callback was triggered by a download. + state_->settings_changed()->Add( + [this](bool downloaded) { + if (!downloaded) { + QueueUpdateSelf(); + } + }); +} + +ContactManager::~ContactManager() { + // Free cache entries. + Clear(&user_cache_); + Clear(&resolved_contact_cache_); +} + +void ContactManager::ProcessMergeAccounts( + const string& op_id, const string& completion_db_key, + const DBHandle& updates) { + if (op_id.empty() || completion_db_key.empty()) { + return; + } + if (updates.get()) { + updates->Put(kMergeAccountsOpIdKey, op_id); + updates->Put(kMergeAccountsCompletionKey, completion_db_key); + // Force a query notification. + state_->notification_manager()->Invalidate(updates); + } + + // We're abusing the fetch contacts infrastructure here, which handles + // watching for an op id in the query notifications stream. + pending_fetch_ops_[op_id].push_back([this, completion_db_key] { + LOG("merge accounts complete"); + DBHandle updates = state_->NewDBTransaction(); + updates->Delete(kMergeAccountsOpIdKey); + updates->Delete(kMergeAccountsCompletionKey); + updates->Delete(completion_db_key); + updates->Commit(); + state_->async()->dispatch_main([this] { + state_->settings_changed()->Run(true); + }); + }); +} + +void ContactManager::ProcessAddressBookImport( + const vector& contacts, + const DBHandle& updates, FetchCallback done) { + CHECK(!dispatch_is_main_thread()); + + StringSet existing; + + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + if (m.contact_source() != kContactSourceIOSAddressBook) { + continue; + } + DCHECK(!m.contact_id().empty()); + existing.insert(m.contact_id()); + } + + const WallTime now = WallTime_Now(); + for (int i = 0; i < contacts.size(); ++i) { + const string contact_id = SaveContact(contacts[i], true, now, updates); + existing.erase(contact_id); + } + + for (StringSet::iterator it = existing.begin(); it != existing.end(); ++it) { + RemoveContact(*it, true, updates); + } + + LOG("contacts: %d contact%s, %d VF, updated %d entr%s, deleted/replaced %d entr%s", + count_, Pluralize(count_), viewfinder_count_, + contacts.size(), Pluralize(contacts.size(), "y", "ies"), + existing.size(), Pluralize(existing.size(), "y", "ies")); + + updates->AddCommitTrigger("ProcessAddressBookImport", [this, done] { + // The next time we reach the end of the upload-contacts queue, schedule our callback. + MutexLock lock(&fetch_ops_mu_); + pending_upload_ops_.push_back(done); + state_->async()->dispatch_main([this] { + state_->net_manager()->Dispatch(); + }); + }); +} + +void ContactManager::ProcessQueryContacts( + const QueryContactsResponse& r, + const ContactSelection& cs, const DBHandle& updates) { + // Validate the contact selection based on queried contacts. + Validate(cs, updates); + + const WallTime now = WallTime_Now(); + for (int i = 0; i < r.contacts_size(); ++i) { + const ContactMetadata& u = r.contacts(i); + + if (u.label_contact_removed()) { + CHECK(u.has_server_contact_id()); + RemoveServerContact(u.server_contact_id(), updates); + continue; + } + + if (!u.has_primary_identity()) { + continue; + } + + SaveContact(u, false, now, updates); + + // If the server gave us any identities with no associated user id, this is our indication that + // that identity is no longer bound to any user, so unlink it if necessary. + // We do this in ProcessQueryContacts instead of SaveContact because the absence of a user id + // in other contexts does not imply that the identity is unbound. + for (int j = 0; j < u.identities_size(); ++j) { + if (!u.identities(j).has_user_id()) { + UnlinkIdentity(u.identities(j).identity(), updates); + } + } + } + + LOG("contacts: %d contact%s, %d VF, updated %d entr%s", + count_, Pluralize(count_), viewfinder_count_, + r.contacts_size(), Pluralize(r.contacts_size(), "y", "ies")); + + updates->AddCommitTrigger("SaveContacts", [this] { + MutexLock lock(&fetch_ops_mu_); + MaybeRunFetchCallbacksLocked(); + }); +} + +void ContactManager::ProcessQueryNotifications( + const QueryNotificationsResponse& r, const DBHandle& updates) { + MutexLock lock(&fetch_ops_mu_); + + for (int i = 0; i < r.notifications_size(); ++i) { + const QueryNotificationsResponse::Notification& n = r.notifications(i); + if (n.has_invalidate()) { + const InvalidateMetadata& invalidate = n.invalidate(); + if (invalidate.has_contacts()) { + Invalidate(invalidate.contacts(), updates); + } + for (int j = 0; j < invalidate.users_size(); ++j) { + InvalidateUser(invalidate.users(j), updates); + } + } + if (n.has_op_id()) { + // The query notifications processing is performed on the network thread, + // the same thread that manipulates the fetch_op_ map in FetchContacts(). + auto ops = pending_fetch_ops_[n.op_id()]; + for (auto it = ops.begin(); it != ops.end(); ++it) { + VLOG("contacts: got notification that op %s is complete; awaiting contact fetch", n.op_id()); + const FetchCallback& done = *it; + updates->AddCommitTrigger( + Format("FetchContacts:%s", n.op_id()), [this, done] { + MutexLock lock(&fetch_ops_mu_); + VLOG("contacts: moving callback from notification to contact queue"); + completed_fetch_ops_.push_back(done); + MaybeRunFetchCallbacksLocked(); + }); + } + pending_fetch_ops_.erase(n.op_id()); + } + } +} + +void ContactManager::ProcessQueryUsers( + const QueryUsersResponse& r, + const vector& q, const DBHandle& updates) { + typedef std::unordered_set UserIdSet; + UserIdSet user_ids(q.begin(), q.end()); + + const WallTime now = WallTime_Now(); + for (int i = 0; i < r.user_size(); ++i) { + ContactMetadata u = r.user(i).contact(); + if (!u.has_user_id()) { + continue; + } + user_ids.erase(u.user_id()); + + // Update the metadata. + SaveUser(u, now, updates); + } + + // Delete any user that was queried but not returned in the + // result. Presumably we don't have permission to retrieve their info and we + // want to avoid looping attempting retrieval that will never succeed. + for (UserIdSet::iterator iter(user_ids.begin()); + iter != user_ids.end(); + ++iter) { + updates->Delete(DBFormat::user_queue_key(*iter)); + } + + LOG("contacts: %d contact%s, %d VF, updated %d entr%s (%d user%s not returned)", + count_, Pluralize(count_), viewfinder_count_, + r.user_size(), Pluralize(r.user_size(), "y", "ies"), + user_ids.size(), Pluralize(user_ids.size())); + + updates->AddCommitTrigger("SaveUsers", [this] { + MutexLock lock(&fetch_ops_mu_); + MaybeRunFetchCallbacksLocked(); + }); + + process_users_.Run(r, q, updates); +} + +void ContactManager::ProcessResolveContact( + const string& identity, const ContactMetadata* metadata) { + { + MutexLock lock(&cache_mu_); + resolving_contacts_.erase(identity); + } + // The network manager runs these callbacks on its thread, but everything else wants to run + // on the main thread. Copy the arguments and dispatch to the main thread. + const ContactMetadata* metadata_copy = NULL; + if (metadata) { + LOG("contacts: resolved identity %s to user_id %d", identity, metadata->user_id()); + MutexLock lock(&cache_mu_); + delete FindOrNull(resolved_contact_cache_, identity); + resolved_contact_cache_[identity] = new ContactMetadata(*metadata); + if (resolved_contact_cache_.size() % 100 == 0) { + LOG("contact: resolved contact cache at %d entries", resolved_contact_cache_.size()); + } + // Copy the metadata for use in the callback + metadata_copy = new ContactMetadata(*metadata); + } else { + LOG("contacts: error resolving identity %s", identity); + } + dispatch_main([this, identity, metadata_copy] { + contact_resolved_.Run(identity, metadata_copy); + delete metadata_copy; + }); +} + +void ContactManager::SearchUsers(const FullTextQuery& query, int search_options, + UserMatchMap* user_matches, StringSet* all_terms) const { + if (query.empty() && (search_options & ALLOW_EMPTY_SEARCH)) { + // Special case handling of empty searches. We could just search for the + // term "", but that would match every name key and be unacceptably slow. + for (DB::PrefixIterator iter(state_->db(), DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + const Slice value = iter.value(); + ContactMetadata c; + if (c.ParseFromArray(value.data(), value.size()) && + IsViewfinderUser(c)) { + ContactMatch& m = (*user_matches)[c.user_id()]; + m.metadata.Swap(&c); + } + } + return; + } + + for (ScopedPtr iter(user_index_->Search(state_->db(), query)); + iter->Valid(); + iter->Next()) { + // The search process gave us ids; map those to ContactMetadata. + const int64_t user_id = FromString(iter->doc_id()); + ContactMetadata m; + if (!LookupUser(user_id, &m)) { + LOG("contacts: unable to lookup contact metadata for id %s", user_id); + continue; + } + if (!IsViewfinderUser(m)) { + continue; + } + ContactMatch& c = (*user_matches)[m.user_id()]; + c.metadata.MergeFrom(m); + iter->GetRawTerms(all_terms); + } +} + +void ContactManager::SearchContacts(const FullTextQuery& query, int search_options, + ContactMatchMap* contact_matches, StringSet* all_terms) const { + if (search_options & VIEWFINDER_USERS_ONLY) { + return; + } + if (query.empty() && (search_options & ALLOW_EMPTY_SEARCH)) { + for (DB::PrefixIterator iter(state_->db(), DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + const Slice value = iter.value(); + ContactMetadata c; + if (c.ParseFromArray(value.data(), value.size())) { + ContactMatch& m = (*contact_matches)[c.contact_id()]; + m.metadata.Swap(&c); + } + } + return; + } + + for (ScopedPtr iter(contact_index_->Search(state_->db(), query)); + iter->Valid(); + iter->Next()) { + const string contact_id = iter->doc_id().as_string(); + ContactMetadata m; + if (!state_->db()->GetProto(DBFormat::contact_key(contact_id), &m)) { + LOG("contacts: unable to lookup contact metadata for contact id %s", contact_id); + continue; + } + if ((search_options & SKIP_FACEBOOK_CONTACTS) && + m.contact_source() == kContactSourceFacebook) { + continue; + } + ContactMatch& c = (*contact_matches)[m.contact_id()]; + c.metadata.MergeFrom(m); + iter->GetRawTerms(all_terms); + } +} + +// The MatchMaps are non-const because we need access to the non-const ContactMatches they contain; +// this method does not modify the maps themselves. +void ContactManager::MergeSearchResults(UserMatchMap* user_matches, ContactMatchMap* contact_matches, + vector* match_vec) const { + // Build up a vector of the matches. Combine users and contacts with heuristic deduping. + StringSet registered_user_identities; + StringSet prospective_user_identities; + for (UserMatchMap::iterator iter(user_matches->begin()); + iter != user_matches->end(); + ++iter) { + ContactMatch* m = &iter->second; + match_vec->push_back(m); + StringSet* identity_set = m->metadata.label_registered() ? + ®istered_user_identities : + &prospective_user_identities; + if (!m->metadata.primary_identity().empty()) { + identity_set->insert(m->metadata.primary_identity()); + } + for (int i = 0; i < m->metadata.identities_size(); i++) { + identity_set->insert(m->metadata.identities(i).identity()); + } + } + + typedef std::unordered_multimap ContactByNameMap; + ContactByNameMap contact_by_name; + for (ContactMatchMap::iterator iter(contact_matches->begin()); + iter != contact_matches->end(); + ++iter) { + ContactMatch* m = &iter->second; + // Hide any contacts for which we have a matching registered user. + // Remove any contact identities that match a prospective user. + if (!m->metadata.primary_identity().empty()) { + if (ContainsKey(registered_user_identities, m->metadata.primary_identity())) { + continue; + } + if (ContainsKey(prospective_user_identities, m->metadata.primary_identity())) { + m->metadata.clear_primary_identity(); + } + } + bool seen_registered_identity = false; + for (int i = 0; i < m->metadata.identities_size(); i++) { + const string& identity = m->metadata.identities(i).identity(); + if (ContainsKey(registered_user_identities, identity)) { + seen_registered_identity = true; + break; + } + if (ContainsKey(prospective_user_identities, identity)) { + ProtoRepeatedFieldRemoveElement(m->metadata.mutable_identities(), i--); + } + } + if (seen_registered_identity || m->metadata.identities_size() == 0) { + continue; + } + if (m->metadata.primary_identity().empty()) { + ChoosePrimaryIdentity(&m->metadata); + } + + // Heuristically dedupe non-user contacts by name. + string name; + if (!m->metadata.name().empty()) { + name = m->metadata.name(); + } else if (!m->metadata.primary_identity().empty()) { + name = IdentityManager::IdentityToName(m->metadata.primary_identity()); + } + if (!name.empty()) { + pair match_by_name( + contact_by_name.equal_range(name)); + ContactMatch* overlap = NULL; + for (ContactByNameMap::iterator i = match_by_name.first; i != match_by_name.second; ++i) { + // Facebook contacts don't convey any useful information (they're only used for the name + // and the user is prompted to enter an email address if they select one), so hide + // them if there are any real contacts with the same name. + if (m->metadata.contact_source() == kContactSourceFacebook && + i->second->metadata.contact_source() != kContactSourceFacebook) { + // We have a non-facebook contact, so skip this one. + continue; + } else if (m->metadata.contact_source() != kContactSourceFacebook && + i->second->metadata.contact_source() == kContactSourceFacebook) { + // This contact can replace an earlier facebook one. + i->second->metadata.Swap(&m->metadata); + continue; + } + + // Non-facebook contacts can be combined if they have at least one identity in common. + if (IdentitiesOverlap(m->metadata, i->second->metadata)) { + overlap = i->second; + break; + } + } + if (overlap != NULL) { + MergeIdentities(m->metadata, &overlap->metadata); + continue; + } + + contact_by_name.insert(ContactByNameMap::value_type(name, m)); + } + + match_vec->push_back(m); + } +} + +void ContactManager::BuildSearchResults(const vector& match_vec, int search_options, + ContactVec* results) const { + // Output the matching contact metadata. + std::unordered_set user_ids; + for (int i = 0; i < match_vec.size(); ++i) { + ContactMetadata* m = &match_vec[i]->metadata; + if ((search_options & VIEWFINDER_USERS_ONLY) && + !m->has_user_id()) { + continue; + } + if ((m->has_user_id() && !user_ids.insert(m->user_id()).second) || + m->has_merged_with() || + m->label_terminated()) { + continue; + } + if (m->has_user_id() && m->user_id() == state_->user_id()) { + // Skip any contact/user records for the current user. This happens at the end of the process + // rather than when this user record is first read so that any matching contact records can be + // merged into it rather than displayed separately. + continue; + } + results->push_back(ContactMetadata()); + results->back().Swap(m); + } +} + +void ContactManager::Search( + const string& query, ContactVec* contacts, + ScopedPtr* filter_re, int search_options) const { + WallTimer timer; + int parse_options = 0; + if (search_options & ContactManager::PREFIX_MATCH) { + parse_options |= FullTextQuery::PREFIX_MATCH; + } + ScopedPtr parsed_query(FullTextQuery::Parse(query, parse_options)); + StringSet all_terms; + + UserMatchMap user_matches; + SearchUsers(*parsed_query, search_options, &user_matches, &all_terms); + + ContactMatchMap contact_matches; + SearchContacts(*parsed_query, search_options, &contact_matches, &all_terms); + + vector match_vec; + MergeSearchResults(&user_matches, &contact_matches, &match_vec); + + // Sort the match vector. + DCHECK_NE((search_options & SORT_BY_RANK) != 0, + (search_options & SORT_BY_NAME) != 0); + if (search_options & SORT_BY_RANK) { + std::sort(match_vec.begin(), match_vec.end(), ContactMatchRankLess(state_)); + } else { + std::sort(match_vec.begin(), match_vec.end(), ContactMatchNameLess()); + } + + BuildSearchResults(match_vec, search_options, contacts); + + + // Build up a regular expression that will match the start of any filter + // terms. + if (filter_re) { + FullTextQueryTermExtractor extractor(&all_terms); + extractor.VisitNode(*parsed_query); + filter_re->reset(FullTextIndex::BuildFilterRE(all_terms)); + } + //LOG("contact: searched for [%s] (with options 0x%x). Found %d results in %f milliseconds", + // query, search_options, contacts->size(), timer.Milliseconds()); +} + +string ContactManager::FirstName(int64_t user_id, bool allow_nickname) { + ContactMetadata c; + if (user_id == 0 || !LookupUser(user_id, &c)) { + return ""; + } + return FirstName(c, allow_nickname); +} + +string ContactManager::FirstName(const ContactMetadata& c, bool allow_nickname) { + if (c.user_id() == state_->user_id()) { + return "You"; + } + if (allow_nickname && !c.nickname().empty()) { + return c.nickname(); + } else if (c.has_first_name()) { + return c.first_name(); + } else if (c.has_name()) { + return c.name(); + } else if (c.has_email()) { + return c.email(); + } else { + string email; + if (ContactManager::EmailForContact(c, &email)) { + return email; + } + string phone; + if (ContactManager::PhoneForContact(c, &phone)) { + return phone; + } + } + return string(); +} + +string ContactManager::FullName(int64_t user_id, bool allow_nickname) { + ContactMetadata c; + if (user_id == 0 || !LookupUser(user_id, &c)) { + return ""; + } + return FullName(c, allow_nickname); +} + +string ContactManager::FullName(const ContactMetadata& c, bool allow_nickname) { + if (c.user_id() == state_->user_id()) { + return "You"; + } + if (allow_nickname && !c.nickname().empty()) { + return c.nickname(); + } else if (c.has_name()) { + return c.name(); + } else if (c.has_first_name()) { + return c.first_name(); + } else if (c.has_email()) { + return c.email(); + } else { + string email; + if (ContactManager::EmailForContact(c, &email)) { + return email; + } + string phone; + if (ContactManager::PhoneForContact(c, &phone)) { + return phone; + } + } + return string(); +} + +vector ContactManager::ViewfinderContacts() { + vector ids; + for (DB::PrefixIterator iter(state_->db(), DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + int64_t v; + if (DecodeUserIdKey(iter.key(), &v)) { + ids.push_back(v); + } + } + sort(ids.begin(), ids.end()); + return ids; +} + +void ContactManager::Reset() { + count_ = 0; + viewfinder_count_ = 0; + + { + MutexLock l(&cache_mu_); + user_cache_.clear(); + } + + { + MutexLock l(&queue_mu_); + queued_update_self_ = false; + queued_update_friend_ = 0; + } +} + +void ContactManager::MaybeQueueUser(int64_t user_id, const DBHandle& updates) { + const string user_queue_key = DBFormat::user_queue_key(user_id); + if (updates->Exists(user_queue_key)) { + // User is already queued for retrieval. + return; + } + ContactMetadata c; + if (updates->GetProto(DBFormat::user_id_key(user_id), &c) && + !c.need_query_user()) { + // We've already retrieved the user metadata. + return; + } + QueueUser(user_id, updates); +} + +void ContactManager::QueueUser(int64_t user_id, const DBHandle& updates) { + const string user_queue_key = DBFormat::user_queue_key(user_id); + updates->Put(user_queue_key, string()); +} + +void ContactManager::ListQueryUsers(vector* ids, int limit) { + for (DB::PrefixIterator iter(state_->db(), DBFormat::user_queue_key()); + iter.Valid(); + iter.Next()) { + int64_t v; + if (RE2::FullMatch(iter.key(), *kQueueRE, &v)) { + ids->push_back(v); + } + if (ids->size() >= limit) { + break; + } + } +} + +void ContactManager::MaybeQueueUploadContacts() { + if (queued_upload_contacts_.get()) { + return; + } + ScopedPtr u(new UploadContacts); + + bool more = false; + for (DB::PrefixIterator iter(state_->db(), DBFormat::contact_upload_queue_key("")); + iter.Valid(); + iter.Next()) { + string contact_id; + if (!RE2::FullMatch(iter.key(), *kContactUploadQueueRE, &contact_id)) { + continue; + } + ContactMetadata m; + if (!state_->db()->GetProto(DBFormat::contact_key(contact_id), &m)) { + continue; + } + if (!IsUploadableContactSource(m.contact_source())) { + LOG("contact: non-upload contact queued: %s; removing from upload queue", iter.key()); + state_->db()->Delete(iter.key()); + continue; + } + if (u->contacts.size() >= kUploadContactsLimit) { + more = true; + break; + } + u->contacts.push_back(m); + } + if (u->contacts.size() > 0) { + u->headers.set_op_id(state_->NewLocalOperationId()); + u->headers.set_op_timestamp(WallTime_Now()); + if (!more) { + // If we are uploading contacts and this is the last batch, attach any pending upload contacts + // to this op id. + MutexLock lock(&fetch_ops_mu_); + const string op_id = EncodeOperationId(state_->device_id(), u->headers.op_id()); + for (auto it = pending_upload_ops_.begin(); it != pending_upload_ops_.end(); ++it) { + pending_fetch_ops_[op_id].push_back(*it); + } + pending_upload_ops_.clear(); + } + queued_upload_contacts_.reset(u.release()); + } else { + // Nothing to do. If there are any pending_upload_ops, we just did an import that added no + // contacts. Go ahead and run any upload callbacks. + MutexLock lock(&fetch_ops_mu_); + for (auto it = pending_upload_ops_.begin(); it != pending_upload_ops_.end(); ++it) { + (*it)(); + } + pending_upload_ops_.clear(); + } +} + +void ContactManager::CommitQueuedUploadContacts(const UploadContactsResponse& resp, bool success) { + CHECK(queued_upload_contacts_.get()); + if (!queued_upload_contacts_.get()) { + // Silence the code analyzer. + return; + } + DBHandle updates(state_->NewDBTransaction()); + const WallTime now = WallTime_Now(); + const vector& contacts = queued_upload_contacts_->contacts; + for (int i = 0; i < contacts.size(); i++) { + updates->Delete(DBFormat::contact_upload_queue_key(contacts[i].contact_id())); + } + if (success) { + if (resp.contact_ids_size() == contacts.size()) { + // Merge the contact ids returned from the server into our local contacts. + for (int i = 0; i < contacts.size(); i++) { + ContactMetadata m(contacts[i]); + m.set_server_contact_id(resp.contact_ids(i)); + SaveContact(m, false, now, updates); + } + } else { + DCHECK(false) << Format("uploaded %s contacts, got %s ids back", contacts.size(), resp.contact_ids_size()); + } + } else { + LOG("contact: failed to upload contacts; will not retry"); + } + updates->Commit(); + queued_upload_contacts_.reset(NULL); +} + +void ContactManager::MaybeQueueRemoveContacts() { + if (queued_remove_contacts_.get()) { + return; + } + ScopedPtr r(new RemoveContacts); + + for (DB::PrefixIterator iter(state_->db(), DBFormat::contact_remove_queue_key("")); + iter.Valid(); + iter.Next()) { + string server_contact_id; + if (!RE2::FullMatch(iter.key(), *kContactRemoveQueueRE, &server_contact_id)) { + continue; + } + r->server_contact_ids.push_back(server_contact_id); + } + if (r->server_contact_ids.size() > 0) { + r->headers.set_op_id(state_->NewLocalOperationId()); + r->headers.set_op_timestamp(WallTime_Now()); + queued_remove_contacts_.reset(r.release()); + } +} + +void ContactManager::CommitQueuedRemoveContacts(bool success) { + CHECK(queued_remove_contacts_.get()); + if (!queued_remove_contacts_.get()) { + // Silence the code analyzer. + return; + } + DBHandle updates(state_->NewDBTransaction()); + const vector& ids = queued_remove_contacts_->server_contact_ids; + for (int i = 0; i < ids.size(); i++) { + updates->Delete(DBFormat::contact_remove_queue_key(ids[i])); + } + if (!success) { + LOG("contact: failed to remove contacts on server; will not retry"); + } + updates->Commit(); + queued_remove_contacts_.reset(NULL); +} + +void ContactManager::Validate(const ContactSelection& cs, const DBHandle& updates) { + ContactSelection existing; + if (updates->GetProto(kContactSelectionKey, &existing)) { + if (cs.start_key() <= existing.start_key()) { + updates->Delete(kContactSelectionKey); + } else { + existing.set_start_key(cs.start_key()); + updates->PutProto(kContactSelectionKey, existing); + } + } +} + +void ContactManager::Invalidate(const ContactSelection& cs, const DBHandle& updates) { + ContactSelection existing; + if (updates->GetProto(kContactSelectionKey, &existing)) { + existing.set_start_key(std::min(existing.start_key(), cs.start_key())); + } else { + existing.set_start_key(cs.start_key()); + } + if (cs.all()) { + // The server has garbage collected its removed-contact tombstones so we must wipe our local + // contact database (except for local contacts we have yet to upload) and re-download everything. + StringSet pending_uploads; + for (DB::PrefixIterator iter(updates, DBFormat::contact_upload_queue_key("")); iter.Valid(); iter.Next()) { + pending_uploads.insert(RemovePrefix(iter.key(), DBFormat::contact_upload_queue_key("")).as_string()); + } + + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); iter.Valid(); iter.Next()) { + const Slice contact_id = RemovePrefix(iter.key(), DBFormat::contact_key("")); + if (!ContainsKey(pending_uploads, contact_id.as_string())) { + RemoveContact(contact_id.as_string(), false, updates); + } + } + + // Note that the all flag itself is not persisted. + existing.set_start_key(""); + } + updates->PutProto(kContactSelectionKey, existing); +} + +void ContactManager::InvalidateAll(const DBHandle& updates) { + ContactSelection all; + all.set_start_key(""); + updates->PutProto(kContactSelectionKey, all); +} + +void ContactManager::InvalidateUser(const UserSelection& us, const DBHandle& updates) { + updates->Put(DBFormat::user_queue_key(us.user_id()), string()); +} + +bool ContactManager::GetInvalidation(ContactSelection* cs) { + return state_->db()->GetProto(kContactSelectionKey, cs); +} + +bool ContactManager::LookupUser(int64_t user_id, ContactMetadata* c) const { + return LookupUser(user_id, c, state_->db()); +} + +bool ContactManager::LookupUser(int64_t user_id, ContactMetadata* c, const DBHandle& db) const { + MutexLock lock(&cache_mu_); + + if (ContainsKey(user_cache_, user_id)) { + c->CopyFrom(*user_cache_[user_id]); + if (!c->has_merged_with()) { + return true; + } + // Fall through if this is a merged user. + } else if (!db->GetProto(DBFormat::user_id_key(user_id), c)) { + // LOG("contacts: %d unable to find contact", user_id); + return false; + } + if (c->has_merged_with()) { + // TODO(spencer): we don't cache this case. Maybe we should. + ContactMetadata merged_with_contact; + if (db->GetProto(DBFormat::user_id_key(c->merged_with()), + &merged_with_contact)) { + c->CopyFrom(merged_with_contact); + return true; + } + } + + // Add fetched value to cache. + user_cache_[user_id] = new ContactMetadata; + user_cache_[user_id]->CopyFrom(*c); + if (user_cache_.size() % 100 == 0) { + LOG("contact: lookup cache at %d entries", user_cache_.size()); + } + return true; +} + +bool ContactManager::LookupUserByIdentity(const string& identity, ContactMetadata* c) const { + return LookupUserByIdentity(identity, c, state_->db()); +} + +bool ContactManager::LookupUserByIdentity(const string& identity, ContactMetadata* c, + const DBHandle& db) const { + int64_t user_id = db->Get(DBFormat::user_identity_key(identity), -1); + if (user_id <= 0) { + return false; + } + return LookupUser(user_id, c, db); +} + +void ContactManager::ResolveContact(const string& identity) { + { + MutexLock lock(&cache_mu_); + if (ContainsKey(resolving_contacts_, identity)) { + return; + } + resolving_contacts_.insert(identity); + } + state_->net_manager()->ResolveContact(identity); +} + +bool ContactManager::IsRegistered(const ContactMetadata& c) { + return c.has_user_id() && c.label_registered(); +} + +bool ContactManager::IsProspective(const ContactMetadata& c) { + return c.has_user_id() && !c.label_registered(); +} + +bool ContactManager::IsResolvableEmail(const Slice& email) { + return RE2::FullMatch(email, *kEmailFullRE); +} + +bool ContactManager::GetEmailIdentity(const ContactMetadata& c, string* identity) { + // The primary identity should be found in the identities list below, but since + // the order of identities is not guaranteed explicitly check the primary first. + if (IdentityManager::IsEmailIdentity(c.primary_identity())) { + if (identity) { + *identity = c.primary_identity(); + } + return true; + } + for (int i = 0; i < c.identities_size(); ++i) { + if (IdentityManager::IsEmailIdentity(c.identities(i).identity())) { + if (identity) { + *identity = c.identities(i).identity(); + } + return true; + } + } + // Last ditch effort is to get email from the contact directly. + if (c.has_email()) { + if (identity) { + *identity = IdentityManager::IdentityForEmail(c.email()); + } + return true; + } + return false; +} + +bool ContactManager::GetPhoneIdentity(const ContactMetadata& c, string* identity) { + // The primary identity should be found in the identities list below, but since + // the order of identities is not guaranteed explicitly check the primary first. + if (IdentityManager::IsPhoneIdentity(c.primary_identity())) { + if (identity) { + *identity = c.primary_identity(); + } + return true; + } + for (int i = 0; i < c.identities_size(); ++i) { + if (IdentityManager::IsPhoneIdentity(c.identities(i).identity())) { + if (identity) { + *identity = c.identities(i).identity(); + } + return true; + } + } + if (c.has_phone()) { + if (identity) { + *identity = IdentityManager::IdentityForPhone(c.phone()); + } + return true; + } + return false; +} + +string ContactManager::FormatName( + const ContactMetadata& c, bool shorten, bool always_include_full) { + if (shorten) { + if (!c.nickname().empty()) { + return c.nickname(); + } else if (c.has_first_name()) { + return c.first_name(); + } else if (c.has_name()) { + return c.name(); + } else if (c.has_email()) { + return c.email(); + } else { + string email; + if (ContactManager::EmailForContact(c, &email)) { + return email; + } + string phone; + if (ContactManager::PhoneForContact(c, &phone)) { + return phone; + } + } + } else { + string name; + if (c.has_name()) { + name = c.name(); + } else if (c.has_first_name()) { + name = c.first_name(); + } else if (c.has_email()) { + name = c.email(); + } else { + string email; + if (ContactManager::EmailForContact(c, &email)) { + name = email; + } else { + string phone; + if (ContactManager::PhoneForContact(c, &phone)) { + name = phone; + } + } + } + if (!name.empty()) { + if (!c.nickname().empty()) { + if (always_include_full) { + return Format("%s (%s)", c.nickname(), name); + } else { + return c.nickname(); + } + } else { + return name; + } + } + } + + return "(Pending Invite)"; +} + +bool ContactManager::FetchFacebookContacts( + const string& access_token, FetchCallback done) { + return state_->net_manager()->FetchFacebookContacts( + access_token, [this, done](const string& op_id) { + WatchForFetchContacts(op_id, done); + }); +} + +bool ContactManager::FetchGoogleContacts( + const string& refresh_token, FetchCallback done) { + return state_->net_manager()->FetchGoogleContacts( + refresh_token, [this, done](const string& op_id) { + WatchForFetchContacts(op_id, done); + }); +} + +void ContactManager::ClearFetchContacts() { + // Clear (finish) any outstanding fetch contacts operations as we're skipping + // notification processing. + VLOG("contacts: clearing fetch callbacks"); + MutexLock lock(&fetch_ops_mu_); + FetchContactsMap old_fetch_ops; + pending_fetch_ops_.swap(old_fetch_ops); + for (FetchContactsMap::iterator iter(old_fetch_ops.begin()); + iter != old_fetch_ops.end(); + ++iter) { + auto ops = iter->second; + for (int i = 0; i < ops.size(); i++) { + state_->async()->dispatch_main(ops[i]); + } + } + FetchContactsList old_fetch_ops_list; + completed_fetch_ops_.swap(old_fetch_ops_list); + for (int i = 0; i < old_fetch_ops_list.size(); i++) { + state_->async()->dispatch_main(old_fetch_ops_list[i]); + } +} + +bool ContactManager::SetMyName( + const string& first, const string& last, const string& name) { + LOG("contact: setting my name to first=\"%s\", last=\"%s\", name=\"%s\"", + first, last, name); + + const string normalized_first = NormalizeWhitespace(first); + const string normalized_last = NormalizeWhitespace(last); + const string normalized_name = NormalizeWhitespace(name); + if (first != normalized_first) { + LOG("contact: first name was changed by normalization: [%s] vs [%s]", + first, normalized_first); + } + if (last != normalized_last) { + LOG("contact: last name was changed by normalization: [%s] vs [%s]", + last, normalized_last); + } + if (name != normalized_name) { + LOG("contact: name was changed by normalization: [%s] vs [%s]", + name, normalized_name); + } + if (normalized_first.empty() && normalized_last.empty()) { + LOG("contact: normalized name is empty; not saving"); + return false; + } + + ContactMetadata new_metadata, metadata; + new_metadata.set_first_name(normalized_first); + new_metadata.set_last_name(normalized_last); + new_metadata.set_name(normalized_name); + + // Set the necessary flags so the new metadata is recognized as authoritative in the merge. + new_metadata.set_user_id(state_->user_id()); + + LookupUser(state_->user_id(), &metadata); + + if (new_metadata.first_name() == metadata.first_name() && + new_metadata.last_name() == metadata.last_name()) { + LOG("contact: normalized name unchanged; not saving"); + return true; + } + + DBHandle updates = state_->NewDBTransaction(); + SaveUser(new_metadata, WallTime_Now(), updates); + updates->Commit(); + + QueueUpdateSelf(); + return true; +} + +void ContactManager::SetFriendNickname(int64_t user_id, const string& nickname) { + LOG("contact: setting friend (%d) nickname to \"%s\"", user_id, nickname); + + const string normalized = NormalizeWhitespace(nickname); + if (nickname != normalized) { + LOG("contact: nickname was changed by normalization: [%s] vs [%s]", + nickname, normalized); + } + + ContactMetadata metadata; + if (!LookupUser(user_id, &metadata)) { + LOG("contact: unable to find contact: %d", user_id); + return; + } + + if (metadata.nickname() == nickname) { + // The nickname is unchanged. + return; + } + + metadata.set_nickname(normalized); + + DBHandle updates = state_->NewDBTransaction(); + SaveUser(metadata, WallTime_Now(), updates); + updates->Commit(); + + QueueUpdateFriend(user_id); +} + +int ContactManager::CountContactsForSource(const string& source) { + int count = 0; + for (DB::PrefixIterator iter(state_->db(), DBFormat::contact_key(source)); + iter.Valid(); + iter.Next()) { + count++; + } + return count; +} + +int ContactManager::CountViewfinderContactsForSource(const string& source) { + int count = 0; + for (DB::PrefixIterator iter(state_->db(), DBFormat::contact_key(source)); + iter.Valid(); + iter.Next()) { + const Slice value = iter.value(); + ContactMetadata c; + if (c.ParseFromArray(value.data(), value.size())) { + for (int i = 0; i < c.identities_size(); ++i) { + const string& identity = c.identities(i).identity(); + int64_t user_id = state_->db()->Get(DBFormat::user_identity_key(identity)); + if (user_id) { + ContactMetadata cm; + if (LookupUser(user_id, &cm) && + IsViewfinderUser(cm)) { + count++; + }; + break; + } + } + } + } + return count; +} + +WallTime ContactManager::GetLastImportTimeForSource(const string& source) { + ContactSourceMetadata metadata; + if (state_->db()->GetProto(DBFormat::contact_source_key(source), &metadata)) { + return metadata.last_import_timestamp(); + } + return 0; +} + +void ContactManager::SetLastImportTimeForSource(const string& source, WallTime timestamp) { + ContactSourceMetadata metadata; + metadata.set_last_import_timestamp(timestamp); + state_->db()->PutProto(DBFormat::contact_source_key(source), metadata); +} + +string ContactManager::ConstructFullName(const string& first, const string& last) { + // TODO(spencer): this needs to be localized. + if (first.empty()) { + return last; + } + if (last.empty()) { + return first; + } + return Format("%s %s", first, last); +} + +void ContactManager::QueueUpdateSelf() { + // Queue the update on the current thread to ease testing. + MutexLock l(&queue_mu_); + + queued_update_self_ = true; + + // Persist this flag to the database so that if the app crashes before + // the network operation completes, we'll try again later. + state_->db()->Put(kQueuedUpdateSelfKey, true); + + // Note that we intentionally do not call dispatch_main() here as we want the + // stack to unwind and locks to be released before Dispatch() is called. + state_->async()->dispatch_main_async([this] { + state_->net_manager()->Dispatch(); + }); + // TODO(marc): maybe we should look for notifications after this, although + // we're the device changing the settings, so we don't need them right away. +} + +void ContactManager::CommitQueuedUpdateSelf() { + MutexLock l(&queue_mu_); + queued_update_self_ = false; + state_->db()->Delete(kQueuedUpdateSelfKey); +} + +void ContactManager::QueueUpdateFriend(int64_t user_id) { + // Queue the update on the current thread to ease testing. + MutexLock l(&queue_mu_); + if (user_id != 0) { + state_->db()->Put(DBFormat::user_update_queue_key(user_id), 0); + } + + if (queued_update_friend_ == 0) { + // We do not have a friend update currently queued. Find the first + // friend update and queue it. + for (DB::PrefixIterator iter(state_->db(), DBFormat::user_update_queue_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + if (!RE2::FullMatch(key, *kUpdateQueueRE, &queued_update_friend_)) { + DCHECK(false); + state_->db()->Delete(key); + continue; + } + break; + } + } + + if (queued_update_friend_ != 0) { + // Note that we intentionally do not call dispatch_main() here as we want the + // stack to unwind and locks to be released before Dispatch() is called. + state_->async()->dispatch_main_async([this] { + state_->net_manager()->Dispatch(); + }); + } +} + +void ContactManager::CommitQueuedUpdateFriend() { + { + MutexLock l(&queue_mu_); + state_->db()->Delete(DBFormat::user_update_queue_key(queued_update_friend_)); + queued_update_friend_ = 0; + } + QueueUpdateFriend(0); +} + +bool ContactManager::EmailForContact(const ContactMetadata& c, string* email) { + string email_identity; + if (ContactManager::GetEmailIdentity(c, &email_identity)) { + *email = IdentityManager::IdentityToName(email_identity); + return true; + } + *email = ""; + return false; +} + +bool ContactManager::PhoneForContact(const ContactMetadata& c, string* phone) { + string phone_identity; + if (ContactManager::GetPhoneIdentity(c, &phone_identity)) { + *phone = IdentityManager::IdentityToName(phone_identity); + return true; + } + *phone = ""; + return false; +} + +int ContactManager::Reachability(const ContactMetadata& c) { + int reachability = 0; + + if (ContactManager::GetEmailIdentity(c, NULL)) { + reachability |= REACHABLE_BY_EMAIL; + } + if (ContactManager::GetPhoneIdentity(c, NULL)) { + reachability |= REACHABLE_BY_SMS; + } + return reachability; +} + +bool ContactManager::ParseFullName( + const string& full_name, string* first, string* last) { + if (full_name.empty()) { + *first = ""; + *last = ""; + return false; + } + Slice src(full_name); + string part; + if (!RE2::FindAndConsume(&src, *kWordUnicodeRE, &part)) { + LOG("contacts: unable to parse name: '%s'", full_name); + *first = ""; + *last = ""; + return false; + } + *first = part; + RE2::FindAndConsume(&src, *kWhitespaceUnicodeRE, &part); + *last = src.ToString(); + return true; +} + +bool ContactManager::NameLessThan(const Slice& a, const Slice& b) { + // Sort empty names after everything else. + if (a.empty()) { + return false; + } + if (b.empty()) { + return true; + } + // Sort letters before anything else. + const bool a_isalpha = IsAlphaUnicode(utfnext(a)); + const bool b_isalpha = IsAlphaUnicode(utfnext(b)); + if (a_isalpha != b_isalpha) { + return a_isalpha; + } + return LocalizedCaseInsensitiveCompare(a, b) < 0; +} + +bool ContactManager::ContactNameLessThan( + const ContactMetadata& a, const ContactMetadata& b) { + // Some of this logic is duplicated in ContactMatchNameLess for speed. + const string a_name = ContactNameForSort(a); + const string b_name = ContactNameForSort(b); + if (a_name != b_name) { + return ContactManager::NameLessThan(a_name, b_name); + } + return a.primary_identity() < b.primary_identity(); +} + +void ContactManager::MaybeParseFirstAndLastNames(ContactMetadata* c) { + if (!c->has_first_name() || !c->has_last_name()) { + string first, last; + if (ParseFullName(c->name(), &first, &last)) { + if (!c->has_first_name()) { + c->set_first_name(first); + } + if (!c->has_last_name()) { + c->set_last_name(last); + } + } + } +} + +void ContactManager::SaveUser(const ContactMetadata& new_metadata, WallTime now, const DBHandle& updates) { + CHECK(new_metadata.user_id()); + const int64_t user_id = new_metadata.user_id(); + + ContactMetadata old_metadata; + const bool existing = updates->GetProto(DBFormat::user_id_key(user_id), &old_metadata); + + ContactMetadata merged; + if (existing && new_metadata.need_query_user()) { + // The new metadata is tentative, and we have some existing metadata. + if (old_metadata.need_query_user()) { + // If the old metadata was also tentative, combine it with the new. + merged.CopyFrom(old_metadata); + merged.MergeFrom(new_metadata); + } else { + // The old metadata is a full user record, so don't clobber it with a tentative one (except for merging + // identities) + LOG("contact: attempting to overwrite full user record with tentative data: %s vs %s", + old_metadata, new_metadata); + merged.CopyFrom(old_metadata); + } + if (new_metadata.label_registered() && !old_metadata.label_registered()) { + merged.clear_creation_timestamp(); // Will be reset below. + } + } else { + // If the new metadata is not tentative, most fields of the new record clobber the old. + merged.CopyFrom(new_metadata); + // The old nickname is carried forward if the new metadata doesn't replace it. + if (old_metadata.has_nickname() && !new_metadata.has_nickname()) { + merged.set_nickname(old_metadata.nickname()); + } + } + + if (!merged.has_creation_timestamp()) { + merged.set_creation_timestamp(now); + } + + // The identities are merged separately from the protobuf CopyFrom/MergeFrom operations. + merged.clear_identities(); + merged.clear_primary_identity(); + + if (old_metadata.has_primary_identity()) { + merged.set_primary_identity(old_metadata.primary_identity()); + } + + // TODO(ben): need some way to remove identities. + std::unordered_set identities; + for (int i = 0; i < old_metadata.identities_size(); i++) { + identities.insert(old_metadata.identities(i).identity()); + } + merged.mutable_identities()->CopyFrom(old_metadata.identities()); + + // Ensure that the primary identity is always present in identities(). + if (!old_metadata.primary_identity().empty() && + !ContainsKey(identities, old_metadata.primary_identity())) { + identities.insert(old_metadata.primary_identity()); + AddIdentityAndSave(&merged, old_metadata.primary_identity(), updates); + } + + for (int i = 0; i < new_metadata.identities_size(); i++) { + const ContactIdentityMetadata& identity = new_metadata.identities(i); + if (ContainsKey(identities, identity.identity())) { + continue; + } + identities.insert(identity.identity()); + AddIdentityAndSave(&merged, identity.identity(), updates); + merged.mutable_identities(merged.identities_size() - 1)->CopyFrom(identity); + } + + if (!new_metadata.primary_identity().empty() && + !ContainsKey(identities, new_metadata.primary_identity())) { + identities.insert(new_metadata.primary_identity()); + AddIdentityAndSave(&merged, new_metadata.primary_identity(), updates); + } + + // Add a notice when a contact becomes a registered user. This applies when either a user with identities + // transitions from unregistered to registered. + // Skip this step until the first refresh has completed so we don't show all contacts as new when + // an existing user syncs a new device. + if (user_id != state_->user_id() && + merged.identities_size() > 0 && + merged.label_registered() && + !old_metadata.label_registered() && + state_->refresh_completed()) { + updates->Put(DBFormat::new_user_key(user_id), ""); + } + + // If we're saving a new user (whether the user became new in this transaction or an earlier one), + // trigger a rescan of dashboard notices to ensure we're displaying the correct name. + // (Prospective users often get updated in two stages: first a query_contacts binds the identity, + // and then query_users gets the name and other information). + if (updates->Exists(DBFormat::new_user_key(user_id))) { + updates->AddCommitTrigger(kNewUserCallbackTriggerKey, [this] { + new_user_callback_.Run(); + }); + } + + // Split up the name field if necessary. + MaybeParseFirstAndLastNames(&merged); + + merged.clear_indexed_names(); + if (existing) { + merged.mutable_indexed_names()->CopyFrom(old_metadata.indexed_names()); + } + UpdateTextIndex(&merged, user_index_.get(), updates); + + updates->PutProto(DBFormat::user_id_key(user_id), merged); + + if (user_id != state_->user_id()) { + if (existing && IsViewfinderUser(old_metadata)) { + viewfinder_count_--; + } + if (IsViewfinderUser(merged)) { + viewfinder_count_++; + } + } + + // If the user has been merged with another, we may need to fetch that user too. + if (merged.has_merged_with()) { + MaybeQueueUser(merged.merged_with(), updates); + } + + // If the user was terminated, make sure that we remove user id from + // all viewpoint follower lists. + if (merged.label_terminated()) { + vector viewpoint_ids; + state_->viewpoint_table()->ListViewpointsForUserId(user_id, &viewpoint_ids, updates); + for (int i = 0; i < viewpoint_ids.size(); ++i) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(viewpoint_ids[i], updates); + if (vh.get()) { + vh->Lock(); + vh->RemoveFollower(user_id); + vh->SaveAndUnlock(updates); + } + } + } + + { + // If this user is cached, update the value. + MutexLock lock(&cache_mu_); + if (ContainsKey(user_cache_, user_id)) { + user_cache_[merged.user_id()]->CopyFrom(merged); + } + updates->Delete(DBFormat::user_queue_key(user_id)); + + state_->day_table()->InvalidateUser(user_id, updates); + } + + updates->AddCommitTrigger(Format("SaveUser:%s", user_id), [this] { + contact_changed_.Run(); + }); +} + +string ContactManager::SaveContact(const ContactMetadata& m, bool upload, WallTime now, const DBHandle& updates) { + ContactMetadata metadata(m); + CHECK(!metadata.has_user_id()); + CHECK(metadata.has_contact_source()); + + StringSet identities; + for (int i = 0; i < metadata.identities_size(); i++) { + const ContactIdentityMetadata& ci = metadata.identities(i); + DCHECK(!ci.identity().empty()); + if (ci.identity().empty()) { + LOG("not saving empty identity %s from contact %s", ci, metadata); + continue; + } + identities.insert(ci.identity()); + + if (ci.has_user_id()) { + // TODO(ben): it would be more efficent when there are multiple identities to make all the + // changes at once (assuming all the identities are bound to the same user, which is expected + // but not guaranteed). + LinkUserIdentity(ci.user_id(), ci.identity(), metadata, updates); + MaybeQueueUser(ci.user_id(), updates); + metadata.mutable_identities(i)->clear_user_id(); + } + } + + if (metadata.primary_identity().empty()) { + ChoosePrimaryIdentity(&metadata); + } else { + CHECK(ContainsKey(identities, m.primary_identity())); + } + + if (!metadata.has_creation_timestamp()) { + metadata.set_creation_timestamp(now); + } + + const string new_contact_id = ComputeContactId(metadata); + metadata.set_contact_id(new_contact_id); + + ContactMetadata old_metadata; + const bool exists = updates->GetProto(DBFormat::contact_key(new_contact_id), &old_metadata); + if (exists) { + // Contacts are addressed by a hash of their contents, so if we have a match the new should be the + // same as the old, with the possible exception of the addition of a server id. + DCHECK_EQ(old_metadata.name(), metadata.name()); + DCHECK_EQ(old_metadata.first_name(), metadata.first_name()); + DCHECK_EQ(old_metadata.last_name(), metadata.last_name()); + DCHECK_EQ(old_metadata.nickname(), metadata.nickname()); + DCHECK_EQ(old_metadata.rank(), metadata.rank()); + DCHECK_EQ(old_metadata.contact_source(), metadata.contact_source()); + + if (!metadata.has_server_contact_id() || + old_metadata.server_contact_id() == metadata.server_contact_id()) { + return new_contact_id; + } + // Continue to add the server id to the existing proto. + // Server contact ids are only supplied by the server, so if we're adding one we shouldn't + // be asked to reupload the same contact. + DCHECK(!upload); + } + + if (!exists) { + // No need to update the text index if we're just adding a server contact id. + UpdateTextIndex(&metadata, contact_index_.get(), updates); + count_++; + } + + updates->PutProto(DBFormat::contact_key(new_contact_id), metadata); + if (metadata.has_server_contact_id()) { + updates->Put(DBFormat::server_contact_id_key(metadata.server_contact_id()), new_contact_id); + } + + const string remove_queue_key = DBFormat::contact_remove_queue_key(new_contact_id); + if (updates->Exists(remove_queue_key)) { + updates->Delete(remove_queue_key); + } + + if (upload && IsUploadableContactSource(metadata.contact_source())) { + updates->Put(DBFormat::contact_upload_queue_key(new_contact_id), ""); + } + + updates->AddCommitTrigger(Format("SaveContact:%s", new_contact_id), [this] { + contact_changed_.Run(); + }); + + return new_contact_id; +} + +void ContactManager::ReindexContact(ContactMetadata* m, const DBHandle& updates) { + UpdateTextIndex(m, contact_index_.get(), updates); + updates->PutProto(DBFormat::contact_key(m->contact_id()), *m); +} + +void ContactManager::RemoveContact(const string& contact_id, bool upload, const DBHandle& updates) { + ContactMetadata m; + if (updates->GetProto(DBFormat::contact_key(contact_id), &m)) { + contact_index_->RemoveTerms(m.mutable_indexed_names(), updates); + + // Cancel any pending upload. + updates->Delete(DBFormat::contact_upload_queue_key(m.contact_id())); + + if (!m.server_contact_id().empty()) { + updates->Delete(DBFormat::server_contact_id_key(m.server_contact_id())); + if (upload) { + updates->Put(DBFormat::contact_remove_queue_key(m.server_contact_id()), ""); + } + } + } + updates->Delete(DBFormat::contact_key(contact_id)); +} + +void ContactManager::RemoveServerContact(const string& server_contact_id, const DBHandle& updates) { + string contact_id; + if (updates->Get(DBFormat::server_contact_id_key(server_contact_id), &contact_id)) { + RemoveContact(contact_id, false, updates); + } +} + +void ContactManager::RemoveUser(int64_t user_id, const DBHandle& updates) { + ContactMetadata m; + if (updates->GetProto(DBFormat::user_id_key(user_id), &m)) { + user_index_->RemoveTerms(m.mutable_indexed_names(), updates); + } + updates->Delete(DBFormat::user_id_key(user_id)); +} + +void ContactManager::ResetAll() { + DBHandle updates = state_->NewDBTransaction(); + ContactSelection nuclear; + nuclear.set_all(true); + Invalidate(nuclear, updates); + updates->Commit(); + state_->async()->dispatch_main_async([this] { + state_->net_manager()->Dispatch(); + }); +} + +void ContactManager::GetNewUsers(ContactVec* new_users) { + for (DB::PrefixIterator iter(state_->db(), DBFormat::new_user_key()); + iter.Valid(); + iter.Next()) { + int64_t user_id; + if (!DecodeNewUserKey(iter.key(), &user_id)) { + continue; + } + ContactMetadata m; + if (LookupUser(user_id, &m)) { + new_users->push_back(m); + } + } +} + +void ContactManager::ResetNewUsers(const DBHandle& updates) { + for (DB::PrefixIterator iter(state_->db(), DBFormat::new_user_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + updates->AddCommitTrigger(kNewUserCallbackTriggerKey, [this] { + new_user_callback_.Run(); + }); +} + +string ContactManager::FormatUserToken(int64_t user_id) { + return Format("%s%d_", kUserTokenPrefix, user_id); +} + +int64_t ContactManager::ParseUserToken(const Slice& token) { + int64_t user_id = 0; + RE2::FullMatch(token, *kUserTokenRE, &user_id); + return user_id; +} + +void ContactManager::GetAutocompleteUsers(const Slice& query, FullTextIndex* index, + vector* results) { + // Find all the matching contacts. + vector contact_vec; + std::unordered_map contacts; + state_->contact_manager()->Search( + query.as_string(), &contact_vec, NULL, + ContactManager::SORT_BY_RANK | ContactManager::VIEWFINDER_USERS_ONLY | ContactManager::PREFIX_MATCH); + for (int i = 0; i < contact_vec.size(); i++) { + contacts[contact_vec[i].user_id()] = &contact_vec[i]; + } + + // Cross-reference the matching contacts with the user tokens. This lets us retrieve ranking information + // on the same scale as the other terms, as well as exclude users who have not posted any photos ( + // since we are only looking at the episode index and not the viewpoint index). + // TODO(ben): it's still possible for contacts to appear in the autocomplete but have no results (if the + // episodes they match are not in the library). We could fix this by creating separate indexes for library + // and non-library episodes. + FullTextIndex::SuggestionResults user_tokens; + index->GetSuggestions(state_->db(), ContactManager::kUserTokenPrefix, &user_tokens); + for (int i = 0; i < user_tokens.size(); i++) { + const int64_t user_id = ParseUserToken(user_tokens[i].second); + if (ContainsKey(contacts, user_id)) { + results->push_back({FormatName(*contacts[user_id], false), + user_id, + user_tokens[i].first}); + } + } +} + +void ContactManager::LinkUserIdentity(int64_t user_id, const string& identity, + const ContactMetadata& contact_template, const DBHandle& updates) { + ContactMetadata m; + if (!updates->GetProto(DBFormat::user_id_key(user_id), &m)) { + // If we haven't seen the user before, create it. + m.MergeFrom(contact_template); + m.set_need_query_user(true); + m.set_user_id(user_id); + m.clear_indexed_names(); + m.clear_contact_id(); + m.clear_rank(); + m.clear_contact_source(); + // The identity to be linked will be merged in at the end of this method. + m.clear_primary_identity(); + m.clear_identities(); + } else { + // The user exists. We've had bugs that cause user data to get lost, so merge basic information + // from the contact if it's available on the contact and not the user. + if (m.need_query_user()) { + if (m.name().empty() && !contact_template.name().empty()) { + m.set_name(contact_template.name()); + } + if (m.first_name().empty() && !contact_template.first_name().empty()) { + m.set_first_name(contact_template.first_name()); + } + if (m.last_name().empty() && !contact_template.last_name().empty()) { + m.set_last_name(contact_template.last_name()); + } + } + } + m.add_identities()->set_identity(identity); + SaveUser(m, WallTime_Now(), updates); +} + +void ContactManager::UnlinkIdentity(const string& identity, const DBHandle& updates) { + ContactMetadata m; + if (!LookupUserByIdentity(identity, &m, updates)) { + return; + } + + CHECK(m.has_user_id()); + + google::protobuf::RepeatedPtrField identities; + for (int i = 0; i < m.identities_size(); i++) { + if (m.identities(i).identity() != identity) { + identities.Add()->CopyFrom(m.identities(i)); + } + } + m.mutable_identities()->Swap(&identities); + + if (m.primary_identity() == identity) { + m.clear_primary_identity(); + if (m.identities_size() > 0) { + ChoosePrimaryIdentity(&m); + } + } + + // SaveUser will attempt to merge the given metadata with what's already on disk, so we + // must first delete the existing data by hand. + updates->Delete(DBFormat::user_identity_key(identity)); + updates->PutProto(DBFormat::user_id_key(m.user_id()), m); + + // Now call SaveUser to update the full-text index and the in-memory cache. + SaveUser(m, WallTime_Now(), updates); +} + +void ContactManager::WatchForFetchContacts(const string& op_id, FetchCallback done) { + if (op_id.empty()) { + state_->async()->dispatch_main(done); + return; + } + MutexLock lock(&fetch_ops_mu_); + VLOG("contacts: enqueuing fetch callback for op %s", op_id); + pending_fetch_ops_[op_id].push_back(done); + + state_->async()->dispatch_main([this] { + state_->net_manager()->Refresh(); + }); +} + +void ContactManager::MaybeRunFetchCallbacksLocked() { + fetch_ops_mu_.AssertHeld(); + ContactSelection cs; + if (GetInvalidation(&cs)) { + // We still have contacts to fetch. + return; + } + vector user_ids; + ListQueryUsers(&user_ids, 1); + if (user_ids.size()) { + // We still have users to fetch. + return; + } + + // We're all caught up; run the callbacks. + FetchContactsList callbacks; + callbacks.swap(completed_fetch_ops_); + if (callbacks.size()) { + VLOG("contacts: finished querying contacts; running %d fetch callbacks", + callbacks.size()); + } + for (int i = 0; i < callbacks.size(); i++) { + dispatch_main(callbacks[i]); + } +} + +void ContactManager::UpdateTextIndex(ContactMetadata* c, FullTextIndex* index, const DBHandle& updates) { + string key; + if (index == contact_index_.get()) { + CHECK(!c->contact_id().empty()); + key = c->contact_id(); + } else { + CHECK(c->user_id()); + key = ToString(c->user_id()); + } + + vector index_terms; + int pos = 0; + + for (int i = 0; i < c->identities_size(); i++) { + const string identity_name = IdentityManager::IdentityToName(c->identities(i).identity()); + if (!identity_name.empty()) { + pos = index->ParseIndexTerms(pos, identity_name, &index_terms); + } + } + + // Only index the name if it's a "real" name, and not just the identity. + if (!c->name().empty() && + c->name() != IdentityManager::IdentityToName(c->primary_identity())) { + pos = index->ParseIndexTerms(pos, c->name(), &index_terms); + } + if (!c->nickname().empty()) { + pos = index->ParseIndexTerms(pos, c->nickname(), &index_terms); + } + + index->UpdateIndex(index_terms, key, "", c->mutable_indexed_names(), updates); +} + +void ContactManager::MergeResolvedContact(const ContactMetadata& c, const DBHandle& updates) { + SaveUser(c, WallTime_Now(), updates); + // resolve_contact currently only returns a subset of the user fields; schedule a fetch + // to get the rest. + QueueUser(c.user_id(), updates); +} + +bool ContactManager::GetCachedResolvedContact(const string& identity, ContactMetadata* metadata) { + MutexLock lock(&cache_mu_); + const ContactMetadata* resolved = FindOrNull(resolved_contact_cache_, identity); + if (resolved) { + metadata->CopyFrom(*resolved); + return true; + } + return false; +} + +string ContactManager::GetContactSourceForIdentity(const string& identity) { + if (IdentityManager::IsEmailIdentity(identity)) { + return kContactSourceGmail; + } else if (IdentityManager::IsFacebookIdentity(identity)) { + return kContactSourceFacebook; + } else if (IdentityManager::IsPhoneIdentity(identity)) { + return kContactSourceIOSAddressBook; + } else { + return kContactSourceManual; + } +} + +void ContactManager::ChoosePrimaryIdentity(ContactMetadata* m) { + if (!m->primary_identity().empty()) { + // Primary identity is already set. + return; + } + int best_score = -1; + int best_pos = -1; + for (int i = 0; i < m->identities_size(); i++) { + const int score = PrimaryIdentityPriority(m->identities(i).identity()); + if (score > best_score) { + best_score = score; + best_pos = i; + m->set_primary_identity(m->identities(i).identity()); + } + } + CHECK_GE(best_pos, 0); + if (best_pos != 0) { + // Put the newly-chosen primary identity first in the list. + m->mutable_identities()->SwapElements(0, best_pos); + } +} + +string ContactManager::ContactNameForSort(const ContactMetadata& m) { + if (!m.nickname().empty()) { + return m.nickname(); + } else if (!m.name().empty()) { + return m.name(); + } else if (!m.primary_identity().empty()) { + return IdentityManager::IdentityToName(m.primary_identity()); + } else if (!m.email().empty()) { + return m.email(); + } else if (!m.phone().empty()) { + return FormatPhoneNumberPrefix(m.phone(), GetPhoneNumberCountryCode()); + } else { + return ""; + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ContactManager.h b/clients/shared/ContactManager.h new file mode 100644 index 0000000..1ca73fd --- /dev/null +++ b/clients/shared/ContactManager.h @@ -0,0 +1,415 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_CONTACT_MANAGER_H +#define VIEWFINDER_CONTACT_MANAGER_H + +#import +#import "Callback.h" +#import "DB.h" +#import "FullTextIndex.h" +#import "ScopedPtr.h" +#import "Server.pb.h" +#import "Utils.h" + +class Analytics; +class AppState; +class ContactMetadata; +class ContactSelection; +class FullTextQuery; +class NotificationManager; +class QueryContactsResponse; +class QueryNotificationsResponse; +class QueryUsersResponse; +class UserSelection; + +class ContactManager { + typedef Callback FetchCallback; + typedef vector FetchContactsList; + // Multimaps apparently do strange things with blocks, so use a map of vectors instead. + typedef std::unordered_map FetchContactsMap; + + public: + typedef vector ContactVec; + + enum { + SORT_BY_RANK = 1 << 0, + SORT_BY_NAME = 1 << 1, + ALLOW_EMPTY_SEARCH = 1 << 2, + VIEWFINDER_USERS_ONLY = 1 << 3, + SKIP_FACEBOOK_CONTACTS = 1 << 4, + PREFIX_MATCH = 1 << 5, + }; + + // Reachability enum. + enum { + REACHABLE_BY_EMAIL = 1 << 0, + REACHABLE_BY_SMS = 1 << 1, + }; + + struct AutocompleteUserInfo { + string name; + int64_t user_id; + int score; + }; + + static const string kContactSourceGmail; + static const string kContactSourceFacebook; + static const string kContactSourceIOSAddressBook; + static const string kContactSourceManual; + + static const string kContactIndexName; + static const string kUserIndexName; + + // Represents the state of a pending upload contacts operation + struct UploadContacts { + OpHeaders headers; + vector contacts; + }; + + struct RemoveContacts { + OpHeaders headers; + vector server_contact_ids; + }; + + // Internal search types. + struct ContactMatch { + ContactMatch() + : sort_key_initialized(false) { + } + ContactMetadata metadata; + string sort_key; + bool sort_key_initialized; + }; + + // A map from user id to ContactMatch. + typedef std::unordered_map UserMatchMap; + + // A map from contact id to ContactMatch. + typedef std::unordered_map ContactMatchMap; + + public: + ContactManager(AppState* state); + ~ContactManager(); + + void ProcessAddressBookImport( + const vector& contacts, + const DBHandle& updates, FetchCallback done); + void ProcessMergeAccounts( + const string& op_id, const string& completion_db_key, + const DBHandle& updates); + void ProcessQueryContacts( + const QueryContactsResponse& r, + const ContactSelection& cs, const DBHandle& updates); + void ProcessQueryNotifications( + const QueryNotificationsResponse& r, const DBHandle& updates); + void ProcessQueryUsers( + const QueryUsersResponse& r, + const vector& user_ids, const DBHandle& updates); + void ProcessResolveContact(const string& identity, const ContactMetadata* metadata); + void Search(const string& query, ContactVec* contacts, + ScopedPtr* filter_re, + int search_options = SORT_BY_RANK) const; + string FirstName(int64_t user_id, bool allow_nickname = true); + string FirstName(const ContactMetadata& c, bool allow_nickname = true); + string FullName(int64_t user_id, bool allow_nickname = true); + string FullName(const ContactMetadata& c, bool allow_nickname = true); + vector ViewfinderContacts(); + void Reset(); + + // Check to see if we already have info for the specified user and queue for + // retrieval if we don't. + void MaybeQueueUser(int64_t user_id, const DBHandle& updates); + + // Unconditionally queue the specified user for retrieval. + void QueueUser(int64_t user_id, const DBHandle& updates); + + // Gets the list of user-ids that need to be queried. + void ListQueryUsers(vector* user_ids, int limit); + + // Updates the list of queued contacts + void MaybeQueueUploadContacts(); + + // Gets the list of contacts that need to be uploaded. + const UploadContacts* queued_upload_contacts() { return queued_upload_contacts_.get(); } + + // Marks the queued contacts as uploaded. + void CommitQueuedUploadContacts(const UploadContactsResponse& resp, bool success); + + void MaybeQueueRemoveContacts(); + const RemoveContacts* queued_remove_contacts() { return queued_remove_contacts_.get(); } + void CommitQueuedRemoveContacts(bool success); + + // Validates queried contacts. + void Validate(const ContactSelection& s, const DBHandle& updates); + + // Invalidates contacts to query. + void Invalidate(const ContactSelection& s, const DBHandle& updates); + + // Clears existing invalidation so that all contacts are re-queried. + void InvalidateAll(const DBHandle& updates); + + // Invalidates user to query. + void InvalidateUser(const UserSelection& us, const DBHandle& updates); + + // Gets the current contact invalidation. Returns true if an + // invalidation is available; false if none. + bool GetInvalidation(ContactSelection* cs); + + // Lookup user by user id; uses in-memory user cache. + bool LookupUser(int64_t user_id, ContactMetadata* c) const; + bool LookupUser(int64_t user_id, ContactMetadata* c, const DBHandle& db) const; + // Lookup user by identity; NOT CACHED. + bool LookupUserByIdentity(const string& identity, ContactMetadata* c) const; + bool LookupUserByIdentity(const string& identity, ContactMetadata* c, const DBHandle& db) const; + + // Attempt to resolve the given identity to a user. This method is asynchronous; callers + // should be listening on contact_resolved(). + void ResolveContact(const string& identity); + + // Returns true if the contact is for a registered (as opposed to + // prospective) user. + static bool IsRegistered(const ContactMetadata& c); + + // Returns true if the contact is for a prospective (as opposed to + // registered) user. + static bool IsProspective(const ContactMetadata& c); + + // Returns true if the given string looks like a valid and potentially complete email. + static bool IsResolvableEmail(const Slice& email); + + // Returns true if any identity in the specified contact metadata is a valid + // email/phone identity. If not NULL, sets "*identity" to the first + // email/phone identity if one is found. + static bool GetEmailIdentity(const ContactMetadata& c, string* identity); + static bool GetPhoneIdentity(const ContactMetadata& c, string* identity); + + // Returns a name for the specified contact metadata, formatted + // according to the "shorten" parameter. The preference is to use + // name or first_name if available and revert to email or phone + // number as necessary. Specify "always_include_full" to format + // full name as a parenthetical suffix in the event that there is a + // nickname (e.g. " ()"). + static string FormatName(const ContactMetadata& c, bool shorten, + bool always_include_full = true); + + // Fetch the contacts for the specified auth service. Returns true if we + // queued an operation to refresh contacts for the specified service. Returns + // false if the network is down or the service isn't valid. + bool FetchFacebookContacts(const string& access_token, FetchCallback done); + bool FetchGoogleContacts(const string& refresh_token, FetchCallback done); + + // Clear any outstanding fetch contacts operations. + void ClearFetchContacts(); + + // Sets the current user's name and queues an update to the network. + // Returns false if the given name is invalid. + bool SetMyName(const string& first, const string& last, const string& name); + + // Construct full name based on first and last. Some locales reverse the + // combination of first/last names. + static string ConstructFullName(const string& first, const string& last); + + // Sets the nickname for the specified user and queues an update to the + // network. + void SetFriendNickname(int64_t user_id, const string& nickname); + + void QueueUpdateSelf(); + void CommitQueuedUpdateSelf(); + bool queued_update_self() const { return queued_update_self_; } + + void QueueUpdateFriend(int64_t user_id); + void CommitQueuedUpdateFriend(); + int64_t queued_update_friend() const { return queued_update_friend_; } + + AppState* state() const { return state_; } + + // contact_changed callbacks are run on an unspecified thread + // whenever a contact is updated. + CallbackSet* contact_changed() { return &contact_changed_; } + + // contact_resolved callbacks are run on the main thread when a ResolveContact finishes. + // The metadata will be NULL if the contact could not be found. + CallbackSet2* contact_resolved() { return &contact_resolved_; } + + // Returns the count of the total number of contacts. + int count() const { return count_; } + + // Returns the count of viewfinder contacts. + int viewfinder_count() const { return viewfinder_count_; } + + // Returns the number of contacts from the given source. + int CountContactsForSource(const string& source); + + // Returns the number of Viewfinder contacts (with user_id set) from given source. + int CountViewfinderContactsForSource(const string& source); + + // Returns the last successful import of this source, or 0 if none is found. + WallTime GetLastImportTimeForSource(const string& source); + void SetLastImportTimeForSource(const string& source, WallTime timestamp); + + // Extracts and sets the value of *email/*phone using contact metadata + // "c". If the email/phone is present, it's returned immediately; otherwise, + // if the email/phone can be extracted from the identity, that's + // returned. Returns true on success; false otherwise. + static bool EmailForContact(const ContactMetadata& c, string* email); + static bool PhoneForContact(const ContactMetadata& c, string* phone); + + // Returns the various means by which the user is reachable outside of + // the Viewfinder platform. The return value is a bitwise-or of the + // reachability enums (e.g. REACHABLE_BY_SMS, REACHABLE_BY_EMAIL). + static int Reachability(const ContactMetadata& c); + + // Parses the first and last name from "full_name". The values, if + // they can be parsed, are stored in *first and *last respectively. + // Returns true if names can be parsed; false otherwise. + static bool ParseFullName(const string& full_name, string* first, string* last); + + + // Comparison function for determining if name "a" is less than name + // "b". Performs a case-insensitive comparison and sorts letters before + // non-letters so that numbers sort after letters. + static bool NameLessThan(const Slice& a, const Slice& b); + + // Comparison function for determining if contact "a" is less than name "b" + // using a name comparison. + static bool ContactNameLessThan( + const ContactMetadata& a, const ContactMetadata& b); + + // Rewrites the full-text index for this contact. Rewrites the indexed_names + // field of this contact, so the metadata must be persisted to the database + // afterwards. + void UpdateTextIndex(ContactMetadata* c, FullTextIndex* index, const DBHandle& updates); + + // Writes the given metadata to the database, merging it with any existing data for + // the same identity. Should be used after a contact_resolved callback if the + // new data needs to be saved. + void MergeResolvedContact(const ContactMetadata& c, const DBHandle& updates); + + // If "identity" has been resolved recently, copy it into *metadata and return true. + bool GetCachedResolvedContact(const string& identity, ContactMetadata* metadata); + + // Returns a value for the contact_source field based on the given identity. This is a heuristic + // used for backwards compatibility until we have added an explicit source to all contacts. + static string GetContactSourceForIdentity(const string& identity); + + // Sets the primary_identity field of *m (if necessary) to the best of the known identities. + static void ChoosePrimaryIdentity(ContactMetadata* m); + + // Returns a single display name for this contact to be used for sorting and related operations + // (such as iOS table headers). Uses the first available field out of nickname, name, and primary_identity. + static string ContactNameForSort(const ContactMetadata& m); + + // TODO(ben): Do these need to be public? + void RemoveUser(int64_t user_id, const DBHandle& updates); + void RemoveServerContact(const string& server_contact_id, const DBHandle& updates); + void RemoveContact(const string& contact_id, bool upload, const DBHandle& updates); + void SaveUser(const ContactMetadata& m, WallTime now, const DBHandle& updates); + + // Returns the assigned contact_id. If the "upload" bool is set for a + // non-uploadable contact source, it is ignored. + // This method is efficient to call if the given contact already exists. + string SaveContact(const ContactMetadata& m, bool upload, WallTime now, const DBHandle& updates); + + // Updates the index for the given contact, which is assumed not to have actually changed. + // (probably only useful from migrations). + void ReindexContact(ContactMetadata* m, const DBHandle& updates); + + // Deletes all contacts and causes them to be re-queried from the server. + void ResetAll(); + + // Returns a list of contacts who have been converted to registered users since + // the last call to ResetNewUsers. + void GetNewUsers(ContactVec* new_users); + void ResetNewUsers(const DBHandle& updates); + CallbackSet* new_user_callback() { return &new_user_callback_; } + + typedef CallbackSet3&, const DBHandle&> ProcessUsersCallback; + ProcessUsersCallback* process_users() { return &process_users_; } + + // Returns a string that can be used in search (in other indexes) to find records related to the given user. + static string FormatUserToken(int64_t user_id); + + // Prefix common to all FormatUserToken() strings. + static const string kUserTokenPrefix; + + // Returns the user id encoded in the token or 0. + static int64_t ParseUserToken(const Slice& token); + + // Adds any users matching the query who also appear in *index to *results. + void GetAutocompleteUsers(const Slice& query, FullTextIndex* index, vector* results); + + private: + static string IdentityToIndexPhrase(const string& identity); + void MaybeParseFirstAndLastNames(ContactMetadata* c); + + void LinkUserIdentity(int64_t user_id, const string& identity, const ContactMetadata& contact_template, + const DBHandle& updates); + // If this identity is linked to a user, unlink it. + void UnlinkIdentity(const string& identity, const DBHandle& updates); + + void WatchForFetchContacts(const string& op_id, FetchCallback done); + + // Run all completed FetchContacts callbacks if we have no more query_contacts calls to make. + void MaybeRunFetchCallbacksLocked(); + + void SearchUsers(const FullTextQuery& query, int search_options, + UserMatchMap* user_matches, StringSet* all_terms) const; + void SearchContacts(const FullTextQuery& query, int search_options, + ContactMatchMap* user_matches, StringSet* all_terms) const; + void MergeSearchResults(UserMatchMap* user_matches, ContactMatchMap* contact_matches, + vector* match_vec) const; + void BuildSearchResults(const vector& match_vec, int search_options, ContactVec* results) const; + + private: + AppState* const state_; + int count_; + int viewfinder_count_; + CallbackSet contact_changed_; + CallbackSet new_user_callback_; + CallbackSet2 contact_resolved_; + ProcessUsersCallback process_users_; + + // "Fetch ops" are an increasingly misnamed mechanism for callbacks to be + // scheduled after our state has been synchronized with the server. + // A single callback will be moved from one of the following data structures + // to another before finally being run. + mutable Mutex fetch_ops_mu_; + // A list of callbacks that are waiting for uploads to be processed. + // When the last upload has been assigned an op id, they are moved to + // pending_fetch_ops_. + FetchContactsList pending_upload_ops_; + // Maps op id to FetchContacts callback. Callbacks are moved from pending to completed when + // a query_notifications for the op id has been processed. + FetchContactsMap pending_fetch_ops_; + // A list of FetchContacts callbacks whose notifications and invalidations we have seen. + // They will be called the next time we have no more query_contacts calls to perform. + FetchContactsList completed_fetch_ops_; + + ScopedPtr queued_upload_contacts_; + ScopedPtr queued_remove_contacts_; + + mutable Mutex cache_mu_; + typedef std::unordered_map UserMetadataCache; + mutable UserMetadataCache user_cache_; + + typedef std::unordered_map ResolvedContactCache; + ResolvedContactCache resolved_contact_cache_; + + typedef std::unordered_set ResolvingContactSet; + ResolvingContactSet resolving_contacts_; + + Mutex queue_mu_; + bool queued_update_self_; + int64_t queued_update_friend_; + + ScopedPtr user_index_; + ScopedPtr contact_index_; +}; + +bool DecodeUserIdKey(Slice key, int64_t* user_id); + +bool IsValidEmailAddress(const Slice& address, string* error); + +#endif // VIEWFINDER_CONTACT_MANAGER_H diff --git a/clients/shared/ContactMetadata.proto b/clients/shared/ContactMetadata.proto new file mode 100644 index 0000000..25713fb --- /dev/null +++ b/clients/shared/ContactMetadata.proto @@ -0,0 +1,74 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +// The ContactMetadata protobuf is used for both contacts and users. Contacts are imported from some +// contact source (mobile contacts, gmail, facebook) and never have the user_id field set. Users +// are retrieved from the server and always have user_id set. When a user and contact both exist for +// the same identity, the user should be used because that is the more authoritative source. +// +// LevelDB schema: +// c/ -> ContactMetadata +// cn/ -> contact_id +// u/ -> ContactMetadata +// ui/ -> user_id +// un/ -> user_id + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ContactMetadataPB"; + +message ContactIdentityMetadata { + optional string identity = 1; + // A label such as "home", "work", "mobile". + optional string description = 2; + + // Not stored, but returned by the server in query_contacts. + optional int64 user_id = 3; +} + +message ContactMetadata { + // The primary identity, usually an Email: identity if one exists. + // Important invariant: primary_identity should always be present in identities. + optional string primary_identity = 1; + // All of the user's identities, including the primary identity. + repeated ContactIdentityMetadata identities = 16; + optional string name = 3; + optional string first_name = 6; + optional string last_name = 8; + optional string nickname = 10; + repeated string indexed_names = 4; + // The first time this contact/user was seen on this device. The creation timestamp is updated when a user + // transitions from prospective to registered. + optional double creation_timestamp = 24; + + // The following fields are only set for contacts. + optional string contact_id = 15; + optional string server_contact_id = 19; + optional int64 rank = 5; + optional string contact_source = 17; + + optional bool label_contact_removed = 22; + + // The following fields are only set for users. + optional int64 user_id = 2; + optional int64 merged_with = 9; + // We usually want to use identities instead of these fields, but sometimes for prospective users + // they are all we have. + optional string email = 7; + optional string phone = 26; + + optional bool label_registered = 20; + optional bool label_terminated = 21; + optional bool label_friend = 23; + optional bool label_system = 25; + + optional bool need_query_user = 18; + + optional bool deprecated_user_name = 11; + optional bool deprecated_user_first_name = 12; + optional bool deprecated_user_last_name = 13; + repeated string deprecated_identities = 14; +} + +message ContactSourceMetadata { + optional double last_import_timestamp = 1; +} diff --git a/clients/shared/ContentIds.proto b/clients/shared/ContentIds.proto new file mode 100644 index 0000000..532f14c --- /dev/null +++ b/clients/shared/ContentIds.proto @@ -0,0 +1,31 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ContentIdsPB"; + +message ActivityId { + optional int64 local_id = 1; + optional string server_id = 2; +} + +message CommentId { + optional int64 local_id = 1; + optional string server_id = 2; +} + +message EpisodeId { + optional int64 local_id = 1; + optional string server_id = 2; +} + +message PhotoId { + optional int64 local_id = 1; + optional string server_id = 2; + optional string DEPRECATED_asset_key = 3; +} + +message ViewpointId { + optional int64 local_id = 1; + optional string server_id = 2; +} diff --git a/clients/shared/ContentTable.cc b/clients/shared/ContentTable.cc new file mode 100644 index 0000000..9b4d560 --- /dev/null +++ b/clients/shared/ContentTable.cc @@ -0,0 +1,17 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "ContentTable.h" +#import "StringUtils.h" + +string EncodeContentKey(const string& prefix, int64_t local_id) { + return prefix + ToString(local_id); +} + +string EncodeContentServerKey(const string& prefix, const string& server_id) { + return prefix + server_id; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ContentTable.h b/clients/shared/ContentTable.h new file mode 100644 index 0000000..15aff84 --- /dev/null +++ b/clients/shared/ContentTable.h @@ -0,0 +1,417 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_CONTENT_TABLE_H +#define VIEWFINDER_CONTENT_TABLE_H + +#import +#import +#import "AppState.h" +#import "DB.h" +#import "Mutex.h" +#import "ScopedHandle.h" +#import "ScopedPtr.h" +#import "ServerId.h" +#import "STLUtils.h" +#import "StringUtils.h" + +string EncodeContentKey(const string& prefix, int64_t local_id); +string EncodeContentServerKey(const string& prefix, const string& server_id); + +// ContentTable is thread-safe and ContentHandle is thread-safe, but +// individual Content objects are not. +template +class ContentTable { + public: + // Content is a subclass of the templated metadata plus in-memory + // state, such as uncommitted changes. The Content class is + // reference counted and ContentTable guarantees there is only one + // Content object for each content id. + class Content : public T { + friend class ContentTable; + friend class ScopedHandle; + + public: + // Saves the content to the database, but maintains lock + // for further modifications. + void Save(const DBHandle& updates) { + CHECK(locked_); + CHECK_EQ(disk_local_id_, this->local_id()); + + if (disk_server_id_ != this->server_id()) { + if (!disk_server_id_.empty()) { + updates->Delete(table_->content_server_key(disk_server_id_)); + } + disk_server_id_ = this->server_id(); + if (!disk_server_id_.empty()) { + updates->Put(table_->content_server_key(disk_server_id_), disk_local_id_); + } + } + + // Call table's save hook. + // NOTE: do not change the ordering here, as EpisodeTable relies on the + // table save hook being called before the superclass save. + table_->SaveContentHook(this, updates); + + // Call superclass save. + T::SaveHook(updates); + + // The superclass save might have changed "this", so output it last. + updates->PutProto(table_->content_key(disk_local_id_), *this); + } + + // Deletes the content from the database and releases lock. + void DeleteAndUnlock(const DBHandle& updates) { + CHECK(locked_); + deleted_ = true; + + updates->Delete(table_->content_server_key(disk_server_id_)); + + // Call table's delete hook. + table_->DeleteContentHook(this, updates); + + // Call superclass save. + T::DeleteHook(updates); + + // The superclass save might have changed "this", so output it last. + updates->Delete(table_->content_key(disk_local_id_)); + + Unlock(); + } + + // Save the content to the database and unlock. + void SaveAndUnlock(const DBHandle& updates) { + Save(updates); + Unlock(); + } + + void Lock() { + CHECK(!this->db_->IsSnapshot()); + mu_.Lock(); + locked_ = true; + } + void Unlock() { + CHECK(!this->db_->IsSnapshot()); + locked_ = false; + mu_.Unlock(); + } + + protected: + // Contents cannot be created or destroyed directly by the end user. Use + // ContentTable::{New,Load}Content(). + Content(ContentTable* table, const DBHandle& db, int64_t id) + : T(table->state_, db, id), + table_(table), + locked_(false), + deleted_(false), + disk_local_id_(id) { + } + ~Content() {} + + // Type T needs to implement these methods: + // int64_t local_id() const; + // const string& server_id() const; + + private: + // Loads the content from the database. + bool Load() { + const string key = table_->content_key(disk_local_id_); + if (!this->db_->GetProto(key, this)) { + if (!this->db_->Exists(key)) { + return false; + } + string contents; + if (!this->db_->Get(key, &contents)) { + LOG("unable to load key %s", key); + } else { + LOG("content key %s corrupt: %s", key, contents); + } + return false; + } + if (disk_local_id_ != this->local_id()) { + LOG("protobuf local id mismatch %d != %d for key %s", + disk_local_id_, this->local_id(), key); + return false; + } + disk_server_id_ = this->server_id(); + // Call superclass load. + return T::Load(); + } + + // Increments the content reference count. Only used by ContentHandle. + void Ref() { + refcount_.Ref(); + } + + // Calls content table to decrement reference count and delete the content + // if this is the last remaining reference. Only used by ContentHandle. + void Unref() { + table_->ReleaseContent(this); + } + + private: + ContentTable* const table_; + AtomicRefCount refcount_; + Mutex mu_; + bool locked_; + bool deleted_; + // The local id as stored on disk. + const int64_t disk_local_id_; + // The server id as stored on disk. + string disk_server_id_; + }; + + typedef ScopedHandle ContentHandle; + friend class Content; + + // Subclasses should override the UpdateState() method to return False + // when the iteration is done. + class ContentIterator { + public: + virtual ~ContentIterator() {} + + // Advance the iterator. Sets done() to true if the end of the contents has + // been reached. + void Next() { + CHECK(!reverse_); + while (!done_) { + iter_->Next(); + if (UpdateState()) { + break; + } + } + } + + void Prev() { + CHECK(reverse_); + while (!done_) { + iter_->Prev(); + if (UpdateState()) { + break; + } + } + } + + Slice key() const { return ToSlice(iter_->key()); } + Slice value() const { return ToSlice(iter_->value()); } + bool done() const { return done_; } + + protected: + ContentIterator(leveldb::Iterator* iter, bool reverse) + : reverse_(reverse), + done_(false), + iter_(iter) { + } + + // Position the iterator at the specified key. Sets done() to + // true if the end of the contents has been reached. + void Seek(const string& key) { + done_ = false; + iter_->Seek(key); + // Reverse iterators are a special case. If the seek takes us + // past the valid range of the iterator, seek to the last + // position instead. + if (reverse_) { + if (!iter_->Valid()) { + iter_->SeekToLast(); + } + if (iter_->Valid() && ToSlice(iter_->key()) > key) { + iter_->Prev(); + } + } + while (!UpdateState()) { + if (reverse_) { + iter_->Prev(); + } else { + iter_->Next(); + } + } + } + + bool UpdateState() { + if (!iter_->Valid()) { + done_ = true; + return true; + } + if (IteratorDone(key())) { + done_ = true; + return true; + } + return UpdateStateHook(key()); + } + + // Returns true if the iterator is finished. + virtual bool IteratorDone(const Slice& key) { return false; } + // Returns true if the key is valid and represents an item + // in the iteration. False to skip. + virtual bool UpdateStateHook(const Slice& key) { return true; } + + protected: + const bool reverse_; + + private: + bool done_; + ScopedPtr iter_; + }; + + public: + // Create a new content, allocating a new local content id. + ContentHandle NewContent(const DBHandle& updates) { + MutexLock l(&mu_); + + const int64_t id = state_->NewLocalOperationId(); + + Content*& a = contents_[id]; + if (!a) { + a = new Content(this, updates, id); + } + return ContentHandle(a); + } + + // Load the specified content, from disk if necessary. + ContentHandle LoadContent(int64_t id, const DBHandle& db) { + MutexLock l(&mu_); + Content* snapshot_content = NULL; + // If a snapshot database, return a new instance. + Content*& a = db->IsSnapshot() ? snapshot_content : contents_[id]; + if (a && a->deleted_) { + // Once the content is deleted, it never transitions back to the + // non-deleted state. + return ContentHandle(); + } + if (!a) { + a = new Content(this, db, id); + // The content has not been initialized yet. Do so now. + if (!a->Load()) { + delete a; + if (!db->IsSnapshot()) { + contents_.erase(id); + } + return ContentHandle(); + } + } + return ContentHandle(a); + } + + ContentHandle LoadContent(const string& server_id, const DBHandle& db) { + const int64_t id = ServerToLocalId(server_id, db); + if (id == -1) { + return ContentHandle(); + } + return LoadContent(id, db); + } + + bool Exists(int64_t id, const DBHandle& db) { + return db->Exists(this->content_key(id)); + } + + // Return a count of the number of referenced contents. + int referenced_contents() const { + MutexLock l(&mu_); + return contents_.size(); + } + + // Lookup the local content id associated with a server content id. Returns + // -1 if no mapping is found. + int64_t ServerToLocalId(const string& server_id, const DBHandle& db) { + return db->Get(content_server_key(server_id), -1); + } + + // Decode content key to local id. + int64_t DecodeContentKey(const Slice& key) { + return FromString(key.substr(content_key_prefix_.size()), 0); + } + + // Decode content server key to server id. + string DecodeContentServerKey(const Slice& key) { + return key.substr(content_server_key_prefix_.size()); + } + + // Database integrity check. Returns whether any repairs were made. + virtual bool FSCK( + bool force, ProgressUpdateBlock progress_update, const DBHandle& updates) { + const int cur_fsck_version = updates->Get(fsck_version_key_, 0); + if (force || + (!fsck_version_key_.empty() && + fsck_version_ > cur_fsck_version)) { + if (progress_update) { + progress_update("Repairing Any Bad Data"); + } + updates->Put(fsck_version_key_, fsck_version_); + FSCKImpl(cur_fsck_version, updates); + return true; + } + return false; + } + + AppState* state() const { return state_; } + + protected: + ContentTable(AppState* state, + const string& content_key_prefix, + const string& content_server_key_prefix, + const int fsck_version = 0, + const string& fsck_version_key = "") + : state_(state), + content_key_prefix_(content_key_prefix), + content_server_key_prefix_(content_server_key_prefix), + fsck_version_(fsck_version), + fsck_version_key_(fsck_version_key) { + // The DB might not have been opened at this point, so don't access it yet. + } + virtual ~ContentTable() { + CHECK_EQ(0, contents_.size()); + } + + virtual void SaveContentHook(T* content, const DBHandle& updates) { + // Does nothing by default. + } + + virtual void DeleteContentHook(T* content, const DBHandle& updates) { + // Does nothing by default. + } + + virtual bool FSCKImpl(int prev_fsck_version, const DBHandle& updates) { + // Does nothing by default. + return false; + } + + private: + void ReleaseContent(Content* content) { + MutexLock l(&mu_); + // The reference count must be unref'd while the mutex is held. + if (content->refcount_.Unref()) { + // If not from a snapshot database, remove from the singleton table. + if (!content->db_->IsSnapshot()) { + contents_.erase(content->disk_local_id_); + } + delete content; + } + } + + string content_key(int64_t local_id) const { + return EncodeContentKey(content_key_prefix_, local_id); + } + string content_server_key(const string& server_id) { + return EncodeContentServerKey(content_server_key_prefix_, server_id); + } + + protected: + AppState* const state_; + + private: + const string content_key_prefix_; + const string content_server_key_prefix_; + const int fsck_version_; + const string fsck_version_key_; + mutable Mutex mu_; + // Map of in-memory contents. + std::unordered_map contents_; +}; + +#endif // VIEWFINDER_CONTENT_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/DB.cc b/clients/shared/DB.cc new file mode 100644 index 0000000..734d897 --- /dev/null +++ b/clients/shared/DB.cc @@ -0,0 +1,879 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// Author: Spencer Kimball. + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import "DB.h" +#import "DBFormat.h" +#import "Logging.h" +#import "ScopedPtr.h" +#import "STLUtils.h" +#import "Utils.h" + +namespace { + +// Disable DBLOG statements in APPSTORE builds as they contain Personally +// Identifiable Information. +#ifdef APPSTORE +#define DBLOG if (0) VLOG +#else +#define DBLOG VLOG +#endif + +const int kWriteBufferSize = 1024 * 1024; // 1 MB + +leveldb::Slice ToDBSlice(const Slice& s) { + return leveldb::Slice(s.data(), s.size()); +} + +class LevelDBLogger : public leveldb::Logger { + public: + virtual void Logv(const char* format, va_list ap) { + // The leveldb threads to not have an autorelease pool set up, but we're + // going to call into logging code that potentially creates autoreleased + // objective-c objects. + dispatch_autoreleasepool([this, format, ap] { + LogvImpl(format, ap); + }); + } + + private: + void LogvImpl(const char* format, va_list ap) { + // First try with a small fixed size buffer. + char space[1024]; + + // It's possible for methods that use a va_list to invalidate the data in + // it upon use. The fix is to make a copy of the structure before using it + // and use that copy instead. + va_list backup_ap; + va_copy(backup_ap, ap); + int result = vsnprintf(space, sizeof(space), format, backup_ap); + va_end(backup_ap); + + if ((result >= 0) && (result < sizeof(space))) { + VLOG("%s", Slice(space, result)); + return; + } + + // Repeatedly increase buffer size until it fits. + int length = sizeof(space); + while (true) { + if (result < 0) { + // Older behavior: just try doubling the buffer size. + length *= 2; + } else { + // We need exactly "result+1" characters. + length = result+1; + } + char* buf = new char[length]; + + // Restore the va_list before we use it again + va_copy(backup_ap, ap); + result = vsnprintf(buf, length, format, backup_ap); + va_end(backup_ap); + + if ((result >= 0) && (result < length)) { + // It fit + VLOG("%s", Slice(buf, result)); + delete[] buf; + return; + } + delete[] buf; + } + } +}; + +class LevelDBEnv : public leveldb::EnvWrapper { + typedef leveldb::EnvWrapper EnvWrapper; + typedef leveldb::FileLock FileLock; + typedef leveldb::RandomAccessFile RandomAccessFile; + typedef leveldb::SequentialFile SequentialFile; + typedef leveldb::Status Status; + typedef leveldb::WritableFile WritableFile; + + public: + LevelDBEnv() + : EnvWrapper(Env::Default()) { + } + +#define LOG_ENV_OP(name, op) \ + Status s = op; \ + if (!s.ok()) { \ + LOG("%s: %s", name, s.ToString()); \ + } \ + return s + + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) { + LOG_ENV_OP("NewSequentialFile", + EnvWrapper::NewSequentialFile(fname, result)); + } + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) { + LOG_ENV_OP("NewRandomAccessFile", + EnvWrapper::NewRandomAccessFile(fname, result)); + } + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) { + LOG_ENV_OP("NewWritableFile", + EnvWrapper::NewWritableFile(fname, result)); + } + +#undef LOG_ENV_OP +}; + +class Snapshot; +class Transaction; + +// Read-write implementation of the DB interface. +class DBImpl : public DB { + public: + DBImpl(const string& dir); + virtual ~DBImpl(); + + bool Open(int cache_size); + void Close(); + + DBHandle NewTransaction(); + DBHandle NewSnapshot(); + + bool IsTransaction() const { return false; } + bool IsSnapshot() const { return false; } + + bool Put(const Slice& key, const Slice& value); + bool PutProto(const Slice& key, + const google::protobuf::MessageLite& message); + bool Get(const Slice& key, string* value); + bool GetProto(const Slice& key, + google::protobuf::MessageLite* message); + bool Exists(const Slice& key); + bool Delete(const Slice& key); + + leveldb::Iterator* NewIterator(); + + void Abandon(bool verbose_logging); + bool Commit(bool verbose_logging); + bool Flush(bool verbose_logging); + bool AddCommitTrigger(const string& key, TriggerCallback trigger) { return false; } + int tx_count() const { return 0; } + + int64_t DiskUsage(); + const string& dir() const { return dir_; } + + void MinorCompaction(); + + private: + friend class Snapshot; + friend class Transaction; + + bool Get(const leveldb::ReadOptions& options, + const Slice& key, string* value); + bool GetProto(const leveldb::ReadOptions& options, const Slice& key, + google::protobuf::MessageLite* message); + bool Exists(const leveldb::ReadOptions& options, const Slice& key); + leveldb::Iterator* NewIterator(const leveldb::ReadOptions& options); + + private: + const string dir_; + ScopedPtr cache_; + ScopedPtr env_; + ScopedPtr logger_; + ScopedPtr db_; +}; + + +// Pending update structure. +enum UpdateType { + UPDATE_WRITE, + UPDATE_DELETE, +}; +struct Update { + UpdateType type; + string value; + Update() + : type(UPDATE_WRITE), value("") { + } + Update(UpdateType t, const Slice& v) + : type(t), value(v.data(), v.size()) { + } +}; +typedef std::map UpdateMap; +typedef std::unordered_map TriggerMap; +typedef std::list TriggerList; + + +// Transactional implementation of the database. TODO(spencer): +// there's no reason to use the underlying WriteBatch object if +// transaction type makes writes visible and we're keeping all of the +// writes in memory anyway. +class Transaction : public DB { + private: + class VisibleWritesIterator : public leveldb::Iterator { + public: + VisibleWritesIterator(Transaction* tx); + + virtual bool Valid() const; + virtual void SeekToFirst(); + virtual void SeekToLast(); + virtual void Seek(const leveldb::Slice& target); + virtual void Next(); + virtual void Prev(); + virtual leveldb::Slice key() const { return key_; } + virtual leveldb::Slice value() const { return value_; } + virtual leveldb::Status status() const { return db_iter_->status(); } + + private: + void UpdateState(bool reverse, UpdateMap::const_iterator updates_iter); + + private: + const UpdateMap& updates_; + ScopedPtr db_iter_; + bool valid_; + leveldb::Slice key_; + leveldb::Slice value_; + }; + + friend class VisibleWritesIterator; + + + public: + Transaction(DBImpl* db); + virtual ~Transaction(); + + bool Open(int cache_size); + void Close(); + + DBHandle NewTransaction(); + DBHandle NewSnapshot(); + + bool IsTransaction() const { return true; } + bool IsSnapshot() const { return false; } + + bool Put(const Slice& key, const Slice& value); + bool PutProto(const Slice& key, + const google::protobuf::MessageLite& message); + bool Get(const Slice& key, string* value); + bool GetProto(const Slice& key, + google::protobuf::MessageLite* message); + bool Exists(const Slice& key); + bool Delete(const Slice& key); + leveldb::Iterator* NewIterator(); + + void Abandon(bool verbose_logging); + bool Commit(bool verbose_logging); + bool Flush(bool verbose_logging); + bool AddCommitTrigger(const string& key, TriggerCallback trigger); + int tx_count() const { return tx_count_; } + + int64_t DiskUsage() { + return db_->DiskUsage(); + } + + const string& dir() const { return db_->dir(); } + + private: + DBImpl* db_; + int tx_count_; + UpdateMap updates_; + TriggerMap triggers_; + TriggerList anonymous_triggers_; +}; + + +// Read-only snapshot implementation of database. +class Snapshot : public DB { + public: + Snapshot(DBImpl* db); + Snapshot(const Snapshot& snapshot); + virtual ~Snapshot(); + + bool Open(int cache_size); + void Close(); + + DBHandle NewTransaction(); + DBHandle NewSnapshot(); + + bool IsTransaction() const { return false; } + bool IsSnapshot() const { return true; } + + bool Put(const Slice& key, const Slice& value); + bool PutProto(const Slice& key, + const google::protobuf::MessageLite& message); + bool Get(const Slice& key, string* value); + bool GetProto(const Slice& key, + google::protobuf::MessageLite* message); + bool Exists(const Slice& key); + bool Delete(const Slice& key); + leveldb::Iterator* NewIterator(); + + void Abandon(bool verbose_logging); + bool Commit(bool verbose_logging); + bool Flush(bool verbose_logging); + bool AddCommitTrigger(const string& key, TriggerCallback trigger) { return false; } + int tx_count() const { return 0; } + + int64_t DiskUsage() { + return db_->DiskUsage(); + } + + const string& dir() const { return db_->dir(); } + + private: + DBImpl *db_; + const leveldb::Snapshot* snapshot_; +}; + + +//// +// DBImpl + +DBImpl::DBImpl(const string& dir) + : dir_(dir) { +} + +DBImpl::~DBImpl() { + Close(); +} + +bool DBImpl::Open(int cache_size) { + CHECK(!db_.get()); + cache_.reset(leveldb::NewLRUCache(cache_size)); + env_.reset(new LevelDBEnv()); + logger_.reset(new LevelDBLogger()); + + leveldb::Options options; + options.create_if_missing = true; + options.write_buffer_size = kWriteBufferSize; + options.max_open_files = 100; + // NOTE(peter): The paranoid_checks option will cause leveldb to fail to + // recover a log file if the tail of the log file is corrupted. And the tail + // of a log file can be corrupted (according to the paranoid_checks + // definition) in "normal" conditions if the app crashes while writing a + // fragmented log record. Searching the interwebs, it looks like the + // paranoid_checks option isn't used very often. + options.paranoid_checks = false; + options.block_cache = cache_.get(); + options.env = env_.get(); + options.info_log = logger_.get(); + + leveldb::DB* db; + leveldb::Status status = leveldb::DB::Open(options, dir_, &db); + if (!status.ok()) { + LOG("%s: unable to open: %s", dir_, status.ToString()); + return false; + } + db_.reset(db); + + string value; + if (db_->GetProperty("leveldb.sstables", &value)) { + VLOG("leveldb sstables:\n%s", value); + } + return true; +} + +void DBImpl::Close() { + db_.reset(NULL); +} + +DBHandle DBImpl::NewTransaction() { + return DBHandle(new Transaction(this)); +} + +DBHandle DBImpl::NewSnapshot() { + return DBHandle(new Snapshot(this)); +} + +bool DBImpl::Put(const Slice& key, const Slice& value) { + CHECK(db_.get()); + DBLOG("put: %s", DBIntrospect::Format(key, value)); + leveldb::Status status = + db_->Put(leveldb::WriteOptions(), ToDBSlice(key), ToDBSlice(value)); + if (!status.ok()) { + DIE("%s: put failed: %s", dir_, status.ToString()); + } + return status.ok(); +} + +bool DBImpl::PutProto(const Slice& key, + const google::protobuf::MessageLite& message) { + return Put(key, message.SerializeAsString()); +} + +bool DBImpl::Get(const Slice& key, string* value) { + leveldb::ReadOptions options; + options.fill_cache = true; + return Get(options, key, value); +} + +bool DBImpl::GetProto(const Slice& key, + google::protobuf::MessageLite* message) { + leveldb::ReadOptions options; + options.fill_cache = true; + return GetProto(options, key, message); +} + +bool DBImpl::Exists(const Slice& key) { + leveldb::ReadOptions options; + options.fill_cache = true; + return Exists(options, key); +} + +bool DBImpl::Delete(const Slice& key) { + CHECK(db_.get()); + DBLOG("del: %s", DBIntrospect::Format(key)); + leveldb::Status status = + db_->Delete(leveldb::WriteOptions(), ToDBSlice(key)); + if (!status.ok()) { + DIE("%s: delete failed: %s", dir_, status.ToString()); + } + return status.ok(); +} + +leveldb::Iterator* DBImpl::NewIterator() { + leveldb::ReadOptions options; + options.fill_cache = true; + return NewIterator(options); +} + +void DBImpl::Abandon(bool verbose_logging) { + DIE("cannot commit non-transactional database"); +} + +bool DBImpl::Commit(bool verbose_logging) { + DIE("cannot commit non-transactional database"); + return false; +} + +bool DBImpl::Flush(bool verbose_logging) { + DIE("cannot commit non-transactional database"); + return false; +} + +int64_t DBImpl::DiskUsage() { + leveldb::Range r; + r.limit = "\xff"; + uint64_t size = 0; + db_->GetApproximateSizes(&r, 1, &size); + return size; +} + +bool DBImpl::Get(const leveldb::ReadOptions& options, + const Slice& key, string* value) { + CHECK(db_.get()); + leveldb::Status status = db_->Get(options, ToDBSlice(key), value); + if (!status.ok() && !status.IsNotFound()) { + // TODO(peter): Rather than immediately exiting, we should present a + // helpful alert to the user and the exit. Tricky part is that we want to + // stop DB processing while the alert is being presented. + DIE("%s: get failed: %s", dir_, status.ToString()); + } + return status.ok(); +} + +bool DBImpl::GetProto(const leveldb::ReadOptions& options, const Slice& key, + google::protobuf::MessageLite* message) { + string value; + if (!Get(options, key, &value)) { + return false; + } + return message->ParseFromString(value); +} + +bool DBImpl::Exists(const leveldb::ReadOptions& options, const Slice& key) { + string tmp; + return Get(options, key, &tmp); +} + +leveldb::Iterator* DBImpl::NewIterator(const leveldb::ReadOptions& options) { + CHECK(db_.get()); + return db_->NewIterator(options); +} + +void DBImpl::MinorCompaction() { + // NOTE(ben): CompactRange flushes the memtable to disk, but because the empty range + // (start==end=="", which is different from using NULL pointers) contains no data + // no other on-disk tables will be compacted. There is a TODO in the leveldb code to + // change this (so that the memtable will only be compacted when it contains data covered + // by the requested range), but hopefully it won't be changed unless they add an + // explicit API to trigger a minor compaction. + leveldb::Slice start, end; + db_->CompactRange(&start, &end); +} + + +//// +// Transaction + +Transaction::Transaction(DBImpl* db) + : db_(db), + tx_count_(0) { +} + +Transaction::~Transaction() { + CHECK_EQ(0, tx_count_); +} + +bool Transaction::Open(int cache_size) { + DIE("cannot open derivative database"); + return false; +} + +void Transaction::Close() { + DIE("cannot close derivative database"); +} + +DBHandle Transaction::NewTransaction() { + return db_->NewTransaction(); +} + +DBHandle Transaction::NewSnapshot() { + return db_->NewSnapshot(); +} + +bool Transaction::Put(const Slice& key, const Slice& value) { + updates_[ToString(key)] = Update(UPDATE_WRITE, value); + ++tx_count_; + return true; +} + +bool Transaction::PutProto(const Slice& key, + const google::protobuf::MessageLite& message) { + return Put(key, message.SerializeAsString()); +} + +bool Transaction::Get(const Slice& key, string* value) { + if (ContainsKey(updates_, ToString(key))) { + if (updates_[ToString(key)].type == UPDATE_DELETE) { + return false; + } + *value = updates_[ToString(key)].value; + return true; + } + return db_->Get(key, value); +} + +bool Transaction::GetProto(const Slice& key, + google::protobuf::MessageLite* message) { + string value; + if (!Get(key, &value)) { + return false; + } + return message->ParseFromString(value); +} + +bool Transaction::Exists(const Slice& key) { + if (ContainsKey(updates_, ToString(key))) { + return updates_[ToString(key)].type != UPDATE_DELETE; + } + return db_->Exists(key); +} + +bool Transaction::Delete(const Slice& key) { + updates_[ToString(key)] = Update(UPDATE_DELETE, ""); + ++tx_count_; + return true; +} + +leveldb::Iterator* Transaction::NewIterator() { + return new VisibleWritesIterator(this); +} + +void Transaction::Abandon(bool verbose_logging) { + if (verbose_logging) { + DBLOG("*** abandoning commit ***"); + for (int index = 0; !updates_.empty(); ++index) { + UpdateMap::iterator iter = updates_.begin(); + const Update& u = iter->second; + if (u.type == UPDATE_WRITE) { + DBLOG("put(%d): %s", index, DBIntrospect::Format(iter->first, u.value)); + } else { + DBLOG("del(%d): %s", index, DBIntrospect::Format(iter->first)); + } + updates_.erase(iter); + } + } else { + updates_.clear(); + } + tx_count_ = 0; + triggers_.clear(); + anonymous_triggers_.clear(); +} + +bool Transaction::Flush(bool verbose_logging) { + leveldb::WriteBatch batch; + for (int index = 0; !updates_.empty(); ++index) { + UpdateMap::iterator iter = updates_.begin(); + const Update& u = iter->second; + if (u.type == UPDATE_WRITE) { + if (verbose_logging) { + DBLOG("put(%d): %s", index, DBIntrospect::Format(iter->first, u.value)); + } else { + DBLOG("put(%d): %s [%d]", index, DBIntrospect::Format(iter->first), u.value.size()); + } + batch.Put(ToDBSlice(iter->first), ToDBSlice(u.value)); + } else { + DBLOG("del(%d): %s", index, DBIntrospect::Format(iter->first)); + batch.Delete(ToDBSlice(iter->first)); + } + updates_.erase(iter); + } + leveldb::Status status = db_->db_->Write(leveldb::WriteOptions(), &batch); + if (!status.ok()) { + DIE("%s: batch put failed: %s", db_->dir(), status.ToString()); + Abandon(false); + return false; + } + tx_count_ = 0; + return true; +} + +bool Transaction::Commit(bool verbose_logging) { + if (!Flush(verbose_logging)) { + return false; + } + for (TriggerMap::iterator iter = triggers_.begin(); + iter != triggers_.end(); + ++iter) { + iter->second(); + } + triggers_.clear(); + for (TriggerList::iterator iter = anonymous_triggers_.begin(); + iter != anonymous_triggers_.end(); + ++iter) { + (*iter)(); + } + anonymous_triggers_.clear(); + return true; +} + +bool Transaction::AddCommitTrigger( + const string& key, TriggerCallback trigger) { + if (!trigger) { + return false; + } + if (key.empty()) { + anonymous_triggers_.push_back(trigger); + } else { + triggers_[key] = trigger; + } + return true; +} + +Transaction::VisibleWritesIterator::VisibleWritesIterator( + Transaction* tx) + : updates_(tx->updates_), + db_iter_(tx->db_->NewIterator()), + valid_(false) { +} + +bool Transaction::VisibleWritesIterator::Valid() const { + return valid_; +} + +void Transaction::VisibleWritesIterator::SeekToFirst() { + db_iter_->SeekToFirst(); + UpdateState(false, updates_.begin()); +} + +void Transaction::VisibleWritesIterator::SeekToLast() { + db_iter_->SeekToLast(); + UpdateMap::const_iterator updates_iter = updates_.end(); + if (updates_iter != updates_.begin()) { + --updates_iter; + } + UpdateState(true, updates_iter); +} + +void Transaction::VisibleWritesIterator::Seek(const leveldb::Slice& target) { + db_iter_->Seek(target); + UpdateMap::const_iterator updates_iter = updates_.lower_bound(target.ToString()); + UpdateState(false, updates_iter); +} + +void Transaction::VisibleWritesIterator::Next() { + if (Valid()) { + const string last_key = key_.ToString(); + while (db_iter_->Valid() && db_iter_->key().ToString() <= last_key) { + db_iter_->Next(); + } + UpdateMap::const_iterator updates_iter = updates_.upper_bound(last_key); + UpdateState(false, updates_iter); + } +} + +void Transaction::VisibleWritesIterator::Prev() { + if (Valid()) { + const string last_key = key_.ToString(); + while (db_iter_->Valid() && db_iter_->key().ToString() >= last_key) { + db_iter_->Prev(); + } + UpdateMap::const_iterator updates_iter = updates_.lower_bound(last_key); + if (updates_iter == updates_.end() || updates_iter->first >= last_key) { + if (updates_iter == updates_.begin()) { + updates_iter = updates_.end(); + } else { + --updates_iter; + } + } + UpdateState(true, updates_iter); + } +} + +void Transaction::VisibleWritesIterator::UpdateState( + bool reverse, UpdateMap::const_iterator updates_iter) { + do { + if (updates_iter != updates_.end() && + (!db_iter_->Valid() || + (reverse ? + Slice(updates_iter->first) >= ToSlice(db_iter_->key()) : + Slice(updates_iter->first) <= ToSlice(db_iter_->key())))) { + // Advance db iterator in case keys are equal. + if (db_iter_->Valid() && Slice(updates_iter->first) == ToSlice(db_iter_->key())) { + if (reverse) { + db_iter_->Prev(); + } else { + db_iter_->Next(); + } + } + const bool is_write = updates_iter->second.type == UPDATE_WRITE; + if (is_write) { + key_ = leveldb::Slice(updates_iter->first); + value_ = leveldb::Slice(updates_iter->second.value); + valid_ = true; + break; + } else { + // On delete, advance iterator. + if (reverse) { + if (updates_iter != updates_.begin()) { + --updates_iter; + } else { + updates_iter = updates_.end(); + } + } else { + ++updates_iter; + } + } + } else if (db_iter_->Valid()) { + key_ = db_iter_->key(); + value_ = db_iter_->value(); + valid_ = true; + break; + } else { + valid_ = false; + break; + } + } while (1); +} + + +//// +// Snapshot + +Snapshot::Snapshot(DBImpl* db) + : db_(db), + snapshot_(db_->db_->GetSnapshot()) { + //LOG("created snapshot %p", this); +} + +Snapshot::Snapshot(const Snapshot& snapshot) + : db_(snapshot.db_), + snapshot_(db_->db_->GetSnapshot()) { + //LOG("created copy snapshot %p", this); +} + +Snapshot::~Snapshot() { + db_->db_->ReleaseSnapshot(snapshot_); + //LOG("deleted snapshot %p", this); +} + +bool Snapshot::Open(int cache_size) { + DIE("cannot open derivative database"); + return false; +} + +void Snapshot::Close() { + DIE("cannot close derivative database"); +} + +DBHandle Snapshot::NewTransaction() { + DIE("snapshot database does not allow transactions"); + return DBHandle(); +} + +DBHandle Snapshot::NewSnapshot() { + return DBHandle(new Snapshot(*this)); +} + +bool Snapshot::Put(const Slice& key, const Slice& value) { + DIE("snapshot database does not allow writes"); + return false; +} + +bool Snapshot::PutProto(const Slice& key, + const google::protobuf::MessageLite& message) { + DIE("snapshot database does not allow writes"); + return false; +} + +bool Snapshot::Get(const Slice& key, string* value) { + leveldb::ReadOptions options; + options.fill_cache = true; + options.snapshot = snapshot_; + return db_->Get(options, key, value); +} + +bool Snapshot::GetProto(const Slice& key, + google::protobuf::MessageLite* message) { + leveldb::ReadOptions options; + options.fill_cache = true; + options.snapshot = snapshot_; + return db_->GetProto(options, key, message); +} + +bool Snapshot::Exists(const Slice& key) { + leveldb::ReadOptions options; + options.fill_cache = true; + options.snapshot = snapshot_; + return db_->Exists(options, key); +} + +bool Snapshot::Delete(const Slice& key) { + DIE("snapshot database does not allow writes"); + return false; +} + +leveldb::Iterator* Snapshot::NewIterator() { + leveldb::ReadOptions options; + options.fill_cache = true; + options.snapshot = snapshot_; + return db_->NewIterator(options); +} + +void Snapshot::Abandon(bool verbose_logging) { + DIE("snapshot database does not allow writes"); +} + +bool Snapshot::Commit(bool verbose_logging) { + DIE("snapshot database does not allow writes"); + return false; +} + +bool Snapshot::Flush(bool verbose_logging) { + DIE("snapshot database does not allow writes"); + return false; +} + +} // namespace + + +DBHandle NewDB(const string& dir) { + return DBHandle(new DBImpl(dir)); +} diff --git a/clients/shared/DB.h b/clients/shared/DB.h new file mode 100644 index 0000000..b093f22 --- /dev/null +++ b/clients/shared/DB.h @@ -0,0 +1,204 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_DB_H +#define VIEWFINDER_DB_H + +#import +#import +#import +#import "Callback.h" +#import "ScopedHandle.h" +#import "StringUtils.h" +#import "ScopedPtr.h" + +namespace google { +namespace protobuf { +class MessageLite; +} // namespace protobuf +} // namespace google + +inline Slice ToSlice(const leveldb::Slice& s) { + return Slice(s.data(), s.size()); +} + +class DB { + friend class ScopedHandle; + + public: + // Facilitates iteration over subset of keys matching specified prefix. + class PrefixIterator { + public: + PrefixIterator(const ScopedHandle& db, const Slice& prefix) { + iter_.reset(db->NewIterator()); + prefix_ = prefix.as_string(); + iter_->Seek(leveldb::Slice(prefix.data(), prefix.size())); + } + + // An iterator is either positioned at a key/value pair or the + // iteration is complete. This method returns false iff the iteration is complete. + bool Valid() const { + return iter_->Valid() && iter_->key().starts_with(leveldb::Slice(prefix_)); + } + + // Moves to the next entry in the source. After this call, done() is + // true iff the iterator was positioned at the last entry in the source. + void Next() { iter_->Next(); } + + // Return the key for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + Slice key() const { return ToSlice(iter_->key()); } + + // Return the value for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + Slice value() const { return ToSlice(iter_->value()); } + + private: + ScopedPtr iter_; + string prefix_; + }; + +protected: + public: + virtual bool Open(int cache_size) = 0; + virtual void Close() = 0; + + // Creates a transactional database which batches all writes + // into an atomic update. The contents may be committed or + // abandoned by calling Commit() or Abandon() respectively. + // + // Writes and deletions to the underlying database done as part of + // this batch are visible to all read methods (e.g., Get, Exists, + // & iterators). + // + // NOTE: The isolation level of the returned database is + // "read-committed". Repeated reads of the same key may yield + // different values in the case of concurrent writers. + // + // The returned DB is NOT thread-safe. + virtual ScopedHandle NewTransaction() = 0; + + // Creates a snapshot of the underlying database. + // + // The returned database is thread-safe and read-only. + virtual ScopedHandle NewSnapshot() = 0; + + virtual bool IsTransaction() const = 0; + virtual bool IsSnapshot() const = 0; + + virtual bool Put(const Slice& key, const Slice& value) = 0; + virtual bool PutProto(const Slice& key, + const google::protobuf::MessageLite& message) = 0; + bool Put(const Slice& key, const string& value) { + return Put(key, Slice(value)); + } + template + bool Put(const Slice& key, const T& value) { + return Put(key, ToString(value)); + } + template + bool Put(const string& key, const T& value) { + return Put(Slice(key), value); + } + + virtual bool Get(const Slice& key, string* value) = 0; + bool Get(const string& key, string* value) { + return Get(Slice(key), value); + } + + virtual bool GetProto(const Slice& key, + google::protobuf::MessageLite* message) = 0; + + template + T Get(const Slice& key, const T& default_value = T()) { + string value; + if (!Get(key, &value)) { + return default_value; + } + T res; + FromString(value, &res); + return res; + } + template + T Get(const string& key, const T& default_value = T()) { + return Get(Slice(key), default_value); + } + template + T Get(const char* key, const T& default_value = T()) { + return Get(Slice(key), default_value); + } + + virtual bool Exists(const Slice& key) = 0; + bool Exists(const string& key) { + return Exists(Slice(key)); + } + virtual bool Delete(const Slice& key) = 0; + bool Delete(const string& key) { + return Delete(Slice(key)); + } + + virtual leveldb::Iterator* NewIterator() = 0; + + // Abandons pending changes. Clears all commit triggers. + virtual void Abandon(bool verbose_logging = false) = 0; + + // Commits pending changes. Invokes all commit triggers and + // clears the callback set. + virtual bool Commit(bool verbose_logging = true) = 0; + + // Commits all pending changes to the database without running commit callbacks. + // This is useful to break up large writes, which are inefficient in leveldb. + // However, calling Flush() gives up transactional atomicity - each flush + // is a separate write which will be visible to other threads while the + // transaction continues. + virtual bool Flush(bool verbose_logging = true) = 0; + + // Trigger callbacks to invoke on Commit(). These are not called on + // Abandon(). Triggers are invoked only once, on first commit. Only + // the latest trigger callback for a specified "key" is retained and + // invoked on commit. Returns true if the trigger was added; false + // otherwise. If "key" is empty, all trigger callbacks are retained. + typedef Callback TriggerCallback; + virtual bool AddCommitTrigger(const string& key, TriggerCallback trigger) = 0; + + // Returns the number of operations currently pending. + virtual int tx_count() const = 0; + + virtual int64_t DiskUsage() = 0; + + virtual const string& dir() const = 0; + + // Prepares for a possible shutdown by writing in-memory data to + // disk in an efficiently-loadable form. Writes are always + // persisted to disk as they happen, but in a log form that requires + // some work to recover at the next launch. Calling MinorCompaction + // before shutting down will speed up the next launch of the app. + virtual void MinorCompaction() { }; + + protected: + virtual ~DB() {} + + private: + void Ref() { + refcount_.Ref(); + } + + void Unref() { + if (refcount_.Unref()) { + delete this; + } + } + + private: + AtomicRefCount refcount_; +}; + +typedef ScopedHandle DBHandle; + +// Creates database using specified directory for data. +DBHandle NewDB(const string& dir); + +#endif // VIEWFINDER_DB_H diff --git a/clients/shared/DBFormat.cc b/clients/shared/DBFormat.cc new file mode 100644 index 0000000..d1d8730 --- /dev/null +++ b/clients/shared/DBFormat.cc @@ -0,0 +1,138 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "Callback.h" +#import "DBFormat.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "STLUtils.h" +#import "WallTime.h" + +namespace { + +class IntrospectMap { + struct IntrospectData { + DBIntrospectCallback key; + DBIntrospectCallback value; + }; + + public: + IntrospectMap() { + // Keys that are already human readable. + const string kNullKeys[] = { + DBFormat::asset_deletion_key(""), + DBFormat::deprecated_contact_name_key(), + DBFormat::contact_remove_queue_key(""), + DBFormat::contact_upload_queue_key(""), + DBFormat::deprecated_full_text_index_comment_key(), + DBFormat::deprecated_full_text_index_episode_key(), + DBFormat::deprecated_full_text_index_viewpoint_key(), + DBFormat::new_user_key(), + DBFormat::placemark_histogram_key(), + DBFormat::placemark_histogram_sort_key(), + DBFormat::server_contact_id_key(""), + DBFormat::deprecated_user_name_key(), + DBFormat::user_queue_key(), + DBFormat::user_update_queue_key(), + }; + for (int i = 0; i < ARRAYSIZE(kNullKeys); ++i) { + Register(kNullKeys[i], NULL, NULL); + } + } + + void Register(const string& prefix, + DBIntrospectCallback key, + DBIntrospectCallback value) { + DCHECK(!ContainsKey(m_, prefix)); + IntrospectData* d = &m_[prefix]; + d->key = key; + d->value = value; + } + + void Unregister(const string& prefix) { + DCHECK(ContainsKey(m_, prefix)); + m_.erase(prefix); + } + + string Format(const Slice& key, const Slice& value) const { + // First check for an exact match. + const string prefix = GetPrefix(key); + const IntrospectData* d = FindPtrOrNull(m_, key.ToString()); + if (d) { + return FormatData(d, prefix, key, value); + } + // Next check for a prefix match. + d = FindPtrOrNull(m_, prefix); + DCHECK(d != NULL); + if (d) { + return FormatData(d, prefix, key, value); + } + // If we couldn't find a match, just return the raw key (not the value + // which is often binary garbage). Note the DCHECK above means we shouldn't + // get here unless somebody was lazy. + return key.ToString(); + } + + private: + static string GetPrefix(const Slice& key) { + Slice::size_type n = key.find('/'); + if (n == string::npos) { + return key.ToString(); + } + return key.substr(0, n + 1).ToString(); + } + + static string FormatData(const IntrospectData* d, const string& prefix, + const Slice& key, const Slice& value) { + const string formatted_key = FormatKey(d, prefix, key); + const string formatted_value = FormatValue(d, value); + if (formatted_value.empty()) { + return formatted_key; + } + return ::Format("%s -> %s", formatted_key, formatted_value); + } + + static string FormatKey(const IntrospectData* d, const string& prefix, + const Slice& key) { + if (d->key) { + const string s = d->key(key); + if (!s.empty()) { + return prefix + s; + } + } + return key.ToString(); + } + + static string FormatValue(const IntrospectData* d, const Slice& value) { + if (!value.empty() && d->value) { + return d->value(value); + } + return string(); + } + + private: + std::unordered_map m_; +}; + +LazyStaticPtr introspect; + +} // namespace + +const string DBIntrospect::kUnhandledValue = ""; + +string DBIntrospect::Format(const Slice& key, const Slice& value) { + return introspect->Format(key, value); +} + +DBRegisterKeyIntrospect::DBRegisterKeyIntrospect( + const Slice& prefix, + const DBIntrospectCallback& key, + const DBIntrospectCallback& value) + : prefix_(prefix.ToString()) { + introspect->Register(prefix_, key, value); +} + +DBRegisterKeyIntrospect::~DBRegisterKeyIntrospect() { + introspect->Unregister(prefix_); +} diff --git a/clients/shared/DBFormat.h b/clients/shared/DBFormat.h new file mode 100644 index 0000000..e8051bd --- /dev/null +++ b/clients/shared/DBFormat.h @@ -0,0 +1,304 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_DB_FORMAT_H +#define VIEWFINDER_DB_FORMAT_H + +#import "Callback.h" +#import "Format.h" +#import "Utils.h" +#import "WallTime.h" + +class DBFormat { + public: + // This list is sorted by value rather than by function name, to make it easier to find non-conflicting prefixes. + static string asset_key(const string& s) { + return "a/" + s; + } + static string activity_key() { + return "ac/"; + } + static string activity_server_key() { + return "acs/"; + } + static string activity_timestamp_key(const string& s) { + return "act/" + s; + } + static string asset_deletion_key(const string& s) { + return "ad/" + s; + } + static string asset_fingerprint_key(const string& s) { + return "af/" + s; + } + static string animated_stat_key(const string& s) { + return "ast/" + s; + } + static string deprecated_asset_reverse_key(const string& s) { + return "ar/" + s; + } + static string contact_key(const string& s) { + return "c/" + s; + } + static string comment_activity_key(const string& s) { + return "ca/" + s; + } + static string compose_autosuggest_key(const string& s) { + return "cas/" + s; + } + static string contact_remove_queue_key(const string& server_contact_id) { + return "ccrq/" + server_contact_id; + } + static string contact_upload_queue_key(const string& s) { + return "ccuq/" + s; + } + static string deprecated_contact_id_key() { + return "ci/"; + } + static string deprecated_contact_id_key(const int64_t i) { + return deprecated_contact_id_key() + ToString(i); + } + static string deprecated_contact_name_key() { + return "cn/"; + } + static string comment_key() { + return "co/"; + } + static string comment_server_key() { + return "cos/"; + } + static string user_queue_key() { + return "cq/"; + } + static string user_queue_key(const int64_t i) { + return user_queue_key() + ToString(i); + } + static string server_contact_id_key(const string& s) { + return "csi/" + s; + } + static string contact_source_key(const string& s) { + return "csm/" + s; + } + static string user_update_queue_key() { + return "cuq/"; + } + static string user_update_queue_key(const int64_t i) { + return user_update_queue_key() + ToString(i); + } + static string day_key(const string& s) { + return "day/" + s; + } + static string db_migration_key(const string& s) { + return "dbm/" + s; + } + static string day_event_key(const string& s) { + return "dev/" + s; + } + static string day_episode_invalidation_key(const string& s) { + return "dis/" + s; + } + static string day_summary_row_key(const string& s) { + return "dsr/" + s; + } + static string episode_key() { + return "e/"; + } + static string episode_event_key(const string& s) { + return "ees/" + s; + } + static string episode_photo_key(const string& s) { + return "ep/" + s; + } + static string episode_activity_key(const string& s) { + return "epa/" + s; + } + static string episode_parent_child_key(const string& s) { + return "epc/" + s; + } + static string episode_selection_key(const string& eps) { + return "eps/" + eps; + } + static string episode_server_key() { + return "es/"; + } + static string episode_timestamp_key(const string& s) { + return "et/" + s; + } + static string deprecated_full_text_index_comment_key() { + return "ftic/"; + } + static string deprecated_full_text_index_episode_key() { + return "ftie/"; + } + static string deprecated_full_text_index_viewpoint_key() { + return "ftiv/"; + } + static string full_text_index_key() { + return "ft/"; + } + static string full_text_index_key(const string& s) { + return "ft/" + s + "/"; + } + static string follower_group_key_deprecated(const string& s) { + return "fg/" + s; + } + static string follower_group_key(const string& s) { + return "fog/" + s; + } + static string follower_viewpoint_key(const string& s) { + return "fvp/" + s; + } + static string image_index_key(const string& s) { + return "ii/" + s; + } + static string local_subscription_key(const string& s) { + return "ls/" + s; + } + static string metadata_key(const string& s) { + return "m/" + s; + } + static string new_user_key() { + return "nu/"; + } + static string new_user_key(int64_t user_id) { + return new_user_key() + ToString(user_id); + } + static string network_queue_key(const string& s) { + return "nq/" + s; + } + static string photo_key() { + return "p/"; + } + static string photo_duplicate_queue_key() { + return "pdq/"; + } + static string photo_episode_key(const string& s) { + return "pe/" + s; + } + static string placemark_histogram_key() { + return "ph/"; + } + static string placemark_histogram_key(const string& s) { + return "ph/" + s; + } + static string placemark_histogram_sort_key() { + return "phs/"; + } + static string placemark_histogram_sort_key(const string& s, int weight) { + return Format("phs/%010d/%s", weight, s); + } + static string placemark_key(const string& s) { + return "pl/" + s; + } + static string photo_path_key(const string& s) { + return "pp/" + s; + } + static string photo_path_access_key(const string& s) { + return "ppa/" + s; + } + static string photo_server_key() { + return "ps/"; + } + static string photo_url_key(const string& s) { + return "pu/" + s; + } + static string quarantined_activity_key(const string& s) { + return "qa/" + s; + } + static string server_subscription_key(const string& s) { + return "ss/" + s; + } + static string trapdoor_event_key() { + return "te/"; + } + static string trapdoor_event_key(int64_t vp_id, const string& event_key) { + return Format("%s%s/%s", trapdoor_event_key(), vp_id, event_key); + } + static string trapdoor_key(const string& s) { + return "trp/" + s; + } + static string user_id_key() { + return "u/"; + } + static string user_id_key(int64_t user_id) { + return user_id_key() + ToString(user_id); + } + static string user_invalidation_key(const string& s) { + return "ui/" + s; + } + static string user_identity_key(const string& s) { + return "uid/" + s; + } + static string deprecated_user_name_key() { + return "un/"; + } + static string viewpoint_key() { + return "v/"; + } + static string viewpoint_conversation_key(const string& s) { + return "vcs/" + s; + } + static string viewpoint_activity_key(const string& vpa) { + return "vpa/" + vpa; + } + static string viewpoint_follower_key(const string& s) { + return "vpf/" + s; + } + static string viewpoint_gc_key(const string& s) { + return "vpgc/" + s; + } + static string viewpoint_invalidation_key(const string& s) { + return "vpi/" + s; + } + static string viewpoint_selection_key(const string& vps) { + return "vps/" + vps; + } + static string viewpoint_scroll_offset_key(const string& s) { + return "vpso/" + s; + } + static string viewpoint_server_key() { + return "vs/"; + } + static string viewpoint_summary_key() { + return "vsum/"; + } +}; + +class DBIntrospect { + public: + // Translates a raw key into a human readable debug key. + static string Format(const Slice& key, const Slice& value = Slice()); + + template + static string FormatProto(const Slice& value) { + Message m; + if (!m.ParseFromArray(value.data(), value.size())) { + return string(); + } + return ToString(m); + } + + // Common format for timestamps in human readable debug keys. + static WallTimeFormat timestamp(WallTime t) { + return WallTimeFormat("%Y-%m-%d-%H-%M-%S", t); + } + + static const string kUnhandledValue; +}; + +using DBIntrospectCallback = Callback; + +// Registers a block to be invoked when DBIntrospect::Format() is +// called. prefix must match the prefix of the key up to and including the "/" +// (e.g. "m/"). +class DBRegisterKeyIntrospect { + public: + DBRegisterKeyIntrospect(const Slice& prefix, + const DBIntrospectCallback& key, + const DBIntrospectCallback& value); + ~DBRegisterKeyIntrospect(); + + private: + const string prefix_; +}; + +#endif // VIEWFINDER_DB_FORMAT_H diff --git a/clients/shared/DBMigration.cc b/clients/shared/DBMigration.cc new file mode 100644 index 0000000..285167f --- /dev/null +++ b/clients/shared/DBMigration.cc @@ -0,0 +1,1382 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import "ActivityTable.h" +#import "ContactManager.h" +#import "CommentTable.h" +#import "DBFormat.h" +#import "DBMigration.h" +#import "EpisodeMetadata.pb.h" +#import "EpisodeTable.h" +#import "IdentityManager.h" +#import "InvalidateMetadata.pb.h" +#import "PeopleRank.h" +#import "PhotoTable.h" +#import "PlacemarkTable.h" +#import "ServerUtils.h" +#import "Timer.h" +#import "ViewpointTable.h" + +namespace { + +// DEPRECATED. +// The enum for migration steps and the migration version key are +// deprecated in favor of using separate keys for each step. Separate +// keys allow the migrations to be added in any order and from +// independent development branches without merge issues. +enum MigrationEnum { + EPISODE_PARENT_CHILD_INDEX = 1, + INITIALIZE_UPLOAD_EPISODE_BIT, + MAYBE_UNQUARANTINE_PHOTOS, +}; +const string kDBMigrationVersionKey = DBFormat::metadata_key("migration_version"); +// End DEPRECATED. + +const string kMigrationKeyPrefix = DBFormat::db_migration_key(""); +// Ordered by putative migration order, not lexicographically. +const string kUseMigrationKeysKey = DBFormat::db_migration_key("use_migration_keys"); +const string kEpisodeParentChildIndexKey = DBFormat::db_migration_key("episode_parent_child_index"); +const string kInitializeUploadEpisodeBitKey = DBFormat::db_migration_key("initialize_update_episode_bit"); +const string kMaybeUnquarantinePhotosKey = DBFormat::db_migration_key("maybe_unquarantine_photos"); +const string kMoveAssetKeysKey = DBFormat::db_migration_key("move_asset_keys"); +const string kMultipleEpisodeActivityIndexKey = DBFormat::db_migration_key("multiple_episode_activity_index"); +const string kRemoveDaySummaryRowsKey = DBFormat::db_migration_key("remove_day_summary_rows"); +const string kRemoveInvalidPlacemarksKey = DBFormat::db_migration_key("remove_invalid_placemarks"); +const string kUploadSavePhotosKey = DBFormat::db_migration_key("upload_save_photos"); +const string kCommentActivityIndexKey = DBFormat::db_migration_key("comment_activity_index"); +// See also kInvalidateContactsKey below. +const string kRequeryContactsKey = DBFormat::db_migration_key("requery_contacts4"); +const string kSplitAssetKeysKey = DBFormat::db_migration_key("split_asset_keys"); +const string kAssetFingerprintIndexKey = DBFormat::db_migration_key("asset_fingerprint_index"); +const string kContactIndexStoreRawKey = DBFormat::db_migration_key("contact_index_store_raw"); +const string kContactIdIndexStoreRawKey = DBFormat::db_migration_key("contact_id_index_store_raw"); +const string kSetCoverPhotosKey = DBFormat::db_migration_key("set_cover_photos5"); +const string kQuarantinedPhotoEpisodeKey = DBFormat::db_migration_key("quarantined_photo_episode"); +const string kSaveComingledLibraryKey = DBFormat::db_migration_key("save_comingled_library"); +const string kSplitContactsUsersKey = DBFormat::db_migration_key("split_contacts_users"); +const string kContactAliasCleanupKey = DBFormat::db_migration_key("contact_alias_cleanup"); +const string kDeleteEmptyContactKey = DBFormat::db_migration_key("delete_empty_contact"); +const string kIndexServerContactIdKey = DBFormat::db_migration_key("index_server_contact_id"); +const string kRequerySelfKey = DBFormat::db_migration_key("requery_self"); +const string kResetNeedQueryUsersKey = DBFormat::db_migration_key("reset_need_query_users2"); +const string kInvalidateContactsKey = DBFormat::db_migration_key("invalidate_contacts"); +const string kCleanupContactIdentitiesKey = DBFormat::db_migration_key("cleanup_contact_identities2"); +const string kMoveRemovedPhotosToHiddenKey = DBFormat::db_migration_key("move_removed_photos_to_hidden"); +const string kBuildFollowerTablesKey = DBFormat::db_migration_key("build_follower_tables"); +const string kCanonicalizeCommentViewpointKey = DBFormat::db_migration_key("canonicalize_comment_viewpoint"); +const string kReindexCommentsKey = DBFormat::db_migration_key("reindex_comments3"); +const string kBuildFollowerGroupsKey = DBFormat::db_migration_key("build_follower_groups3"); +const string kReindexEpisodesKey = DBFormat::db_migration_key("reindex_episodes8"); +const string kReindexViewpointsKey = DBFormat::db_migration_key("reindex_viewpoints4"); +const string kDeleteIdlessContactsKey = DBFormat::db_migration_key("delete_idless_contacts"); +const string kReindexContactsKey = DBFormat::db_migration_key("reindex_contacts5"); +const string kReindexUsersKey = DBFormat::db_migration_key("reindex_users2"); +const string kRemoveTerminatedFollowersKey = DBFormat::db_migration_key("remove_terminated_followers"); +const string kRemoveFeedEventDataKey = DBFormat::db_migration_key("remove_feed_event_data"); +const string kRemoveLocalOnlyPhotosKey = DBFormat::db_migration_key("remove_local_only_photos"); +const string kConvertAssetFingerprintsKey = DBFormat::db_migration_key("convert_asset_fingerprints"); +const string kIndexPhotosKey = DBFormat::db_migration_key("index_photos"); +const string kRequeryUsersKey = DBFormat::db_migration_key("requery_users2"); +const string kPrepareViewpointGCQueueKey = DBFormat::db_migration_key("prepare_viewpoint_gc_queue"); +const string kRemoveAssetDuplicatePhotosKey = DBFormat::db_migration_key("remove_asset_duplicate_photos_key"); + +const DBRegisterKeyIntrospect kMigrationKeyIntrospect( + kMigrationKeyPrefix, + [](Slice key) { + return key.ToString(); + }, NULL); + +void AddUniqueUser(AppState* state, + int64_t user_id, + std::unordered_set* unique_users, + vector* users) { + if (!user_id || ContainsKey(*unique_users, user_id)) { + return; + } + unique_users->insert(user_id); + ContactMetadata cm; + if (!state->contact_manager()->LookupUser(user_id, &cm)) { + LOG("failed to lookup user %d", user_id); + return; + } + + if (cm.label_terminated()) { + return; + } + if (cm.user_id() != user_id) { + // The user id changed; we must have followed a merged_with link. + // Put both source and target user ids in the deduping set. + unique_users->insert(cm.user_id()); + } + users->push_back(cm); +} + +void ListParticipants(AppState* state, const ViewpointHandle& vh, + vector* participants, + const DBHandle& db) { + std::unordered_set unique_participants; + if (!vh->provisional()) { + // Add viewpoint owner as first participant, but only if this viewpoint is + // not provisional. + AddUniqueUser(state, state->user_id(), &unique_participants, participants); + } + + // Next, add all users who were added during share_new or add_followers activities. + for (ScopedPtr iter( + state->activity_table()->NewViewpointActivityIterator( + vh->id().local_id(), 0, false, db)); + !iter->done(); + iter->Next()) { + ActivityHandle ah = iter->GetActivity(); + + // merge_accounts activities contain a single user id. + if (ah->has_merge_accounts()) { + AddUniqueUser(state, ah->merge_accounts().target_user_id(), &unique_participants, participants); + } + + // add_followers and share_new activities have a list of users. + const ::google::protobuf::RepeatedPtrField* contacts = NULL; + if (ah->has_add_followers()) { + contacts = &ah->add_followers().contacts(); + } else if (ah->has_share_new()) { + contacts = &ah->share_new().contacts(); + } + if (contacts) { + for (int i = 0; i < contacts->size(); ++i) { + ContactMetadata cm = contacts->Get(i); + if (cm.has_user_id()) { + AddUniqueUser(state, cm.user_id(), &unique_participants, participants); + } else { + // It's a local prospective user that hasn't made it to the server, so just use + // the metadata we have directly. + participants->push_back(cm); + } + } + } + } +} + +} // namespace + +DBMigration::DBMigration(AppState* state, ProgressUpdateBlock progress_update) + : state_(state), + progress_update_(progress_update), + migrated_(false) { +} + +DBMigration::~DBMigration() { +} + +bool DBMigration::MaybeMigrate() { + DBHandle updates = state_->NewDBTransaction(); + + RunMigration(kUseMigrationKeysKey, + &DBMigration::UseMigrationKeys, + updates); + RunMigration(kEpisodeParentChildIndexKey, + &DBMigration::EpisodeParentChildIndex, + updates); + RunMigration(kInitializeUploadEpisodeBitKey, + &DBMigration::InitializeUploadEpisodeBit, + updates); + RunMigration(kMaybeUnquarantinePhotosKey, + &DBMigration::MaybeUnquarantinePhotos, + updates); + RunMigration(kMoveAssetKeysKey, + &DBMigration::MoveAssetKeys, + updates); + RunMigration(kMultipleEpisodeActivityIndexKey, + &DBMigration::MultipleEpisodeActivityIndex, + updates); + RunMigration(kRemoveDaySummaryRowsKey, + &DBMigration::RemoveDaySummaryRows, + updates); + RunMigration(kRemoveInvalidPlacemarksKey, + &DBMigration::RemoveInvalidPlacemarks, + updates); + RunMigration(kUploadSavePhotosKey, + &DBMigration::UploadSavePhotos, + updates); + RunMigration(kCommentActivityIndexKey, + &DBMigration::CommentActivityIndex, + updates); + RunMigration(kRequeryContactsKey, + &DBMigration::RequeryContacts, + updates); + RunMigration(kSplitAssetKeysKey, + &DBMigration::SplitAssetKeys, + updates); + RunMigration(kAssetFingerprintIndexKey, + &DBMigration::AssetFingerprintIndex, + updates); + RunMigration(kContactIndexStoreRawKey, + &DBMigration::ContactIndexStoreRaw, + updates); + RunMigration(kContactIdIndexStoreRawKey, + &DBMigration::ContactIdIndexStoreRaw, + updates); + RunMigration(kSetCoverPhotosKey, + &DBMigration::SetCoverPhotos, + updates); + RunMigration(kQuarantinedPhotoEpisodeKey, + &DBMigration::QuarantinedPhotoEpisode, + updates); + RunMigration(kSaveComingledLibraryKey, + &DBMigration::SaveComingledLibrary, + updates); + RunMigration(kSplitContactsUsersKey, + &DBMigration::SplitContactsUsers, + updates); + RunMigration(kContactAliasCleanupKey, + &DBMigration::ContactAliasCleanup, + updates); + RunMigration(kDeleteEmptyContactKey, + &DBMigration::DeleteEmptyContact, + updates); + RunMigration(kIndexServerContactIdKey, + &DBMigration::IndexServerContactId, + updates); + RunMigration(kRequerySelfKey, + &DBMigration::RequerySelf, + updates); + RunMigration(kResetNeedQueryUsersKey, + &DBMigration::ResetNeedQueryUser, + updates); + RunMigration(kInvalidateContactsKey, + &DBMigration::InvalidateContacts, + updates); + RunMigration(kCleanupContactIdentitiesKey, + &DBMigration::CleanupContactIdentities, + updates); + RunMigration(kMoveRemovedPhotosToHiddenKey, + &DBMigration::MoveRemovedPhotosToHidden, + updates); + RunMigration(kBuildFollowerTablesKey, + &DBMigration::BuildFollowerTables, + updates); + RunMigration(kCanonicalizeCommentViewpointKey, + &DBMigration::CanonicalizeCommentViewpoint, + updates); + RunMigration(kReindexCommentsKey, + &DBMigration::ReindexComments, + updates); + RunMigration(kBuildFollowerGroupsKey, + &DBMigration::BuildFollowerGroups, + updates); + RunMigration(kReindexEpisodesKey, + &DBMigration::ReindexEpisodes, + updates); + RunMigration(kReindexViewpointsKey, + &DBMigration::ReindexViewpoints, + updates); + RunMigration(kDeleteIdlessContactsKey, + &DBMigration::DeleteIdlessContacts, + updates); + RunMigration(kReindexContactsKey, + &DBMigration::ReindexContacts, + updates); + RunMigration(kReindexUsersKey, + &DBMigration::ReindexUsers, + updates); + RunMigration(kRemoveTerminatedFollowersKey, + &DBMigration::RemoveTerminatedFollowers, + updates); + RunMigration(kRemoveFeedEventDataKey, + &DBMigration::RemoveFeedEventData, + updates); + RunIOSMigration("7.0", NULL, kRemoveLocalOnlyPhotosKey, + &DBMigration::RemoveLocalOnlyPhotos, + updates); + RunIOSMigration(NULL, "7.0", kConvertAssetFingerprintsKey, + &DBMigration::ConvertAssetFingerprints, + updates); + RunIOSMigration(NULL, NULL, kIndexPhotosKey, + &DBMigration::IndexPhotos, + updates); + RunMigration(kRequeryUsersKey, + &DBMigration::RequeryUsers, + updates); + RunMigration(kPrepareViewpointGCQueueKey, + &DBMigration::PrepareViewpointGCQueue, + updates); + RunIOSMigration(NULL, NULL, kRemoveAssetDuplicatePhotosKey, + &DBMigration::RemoveAssetDuplicatePhotos, + updates); + + if (updates->tx_count() > 0) { + migrated_ = true; + } + updates->Commit(false); + return migrated_; +} + +void DBMigration::RunMigration( + const string& migration_key, migration_func migrator, + const DBHandle& updates) { + if (!updates->Exists(migration_key)) { + if (progress_update_) { + progress_update_("Upgrading Data to Latest Version"); + } + LOG("migrate: running %s", migration_key); + (this->*migrator)(updates); + updates->Put(migration_key, ""); + } + // Flush the transaction to disk, but do not run commit callbacks until + // all migrations have finished. It's definitely safe to flush here + // between migrations; idempotent migrations may define their own + // flush points. + MaybeFlush(updates); +} + +void DBMigration::MaybeFlush(const DBHandle& updates) { + if (updates->tx_count() > 1000) { + migrated_ = true; + updates->Flush(false); + } +} + +void DBMigration::UseMigrationKeys(const DBHandle& updates) { + const int cur_version = updates->Get(kDBMigrationVersionKey, 0); + if (cur_version > EPISODE_PARENT_CHILD_INDEX) { + updates->Put(kEpisodeParentChildIndexKey, ""); + } + if (cur_version > INITIALIZE_UPLOAD_EPISODE_BIT) { + updates->Put(kInitializeUploadEpisodeBitKey, ""); + } + if (cur_version > MAYBE_UNQUARANTINE_PHOTOS) { + updates->Put(kMaybeUnquarantinePhotosKey, ""); + } +} + +void DBMigration::EpisodeParentChildIndex(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const int64_t ep_id = state_->episode_table()->DecodeContentKey(iter.key()); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ep_id, updates); + if (eh->has_parent_id()) { + eh->Lock(); + eh->SaveAndUnlock(updates); + } + } +} + +void DBMigration::InitializeUploadEpisodeBit(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const int64_t ep_id = state_->episode_table()->DecodeContentKey(iter.key()); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ep_id, updates); + bool need_upload = true; + // If shared from another user, obviously must be uploaded. + if (eh->has_parent_id() && eh->user_id() != state_->user_id()) { + need_upload = false; + } else if (eh->has_viewpoint_id()) { + // If has viewpoint id and is the default viewpoint, must be uploaded. + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + eh->viewpoint_id(), updates); + if (vh.get() && vh->is_default()) { + need_upload = false; + } + } + if (need_upload) { + eh->Lock(); + eh->set_upload_episode(true); + eh->SaveAndUnlock(updates); + } + } +} + +void DBMigration::MaybeUnquarantinePhotos(const DBHandle& updates) { + std::unordered_map unquarantined; + + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const int64_t ep_id = state_->episode_table()->DecodeContentKey(iter.key()); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ep_id, updates); + if (eh->id().has_server_id()) { + bool episode_invalidated = false; + vector photo_ids; + eh->ListAllPhotos(&photo_ids); + for (int i = 0; i < photo_ids.size(); ++i) { + // Skip removed && hidden photos. + if (eh->IsHidden(photo_ids[i]) || eh->IsRemoved(photo_ids[i])) { + continue; + } + + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_ids[i], updates); + if (ph->label_error() && + (ph->error_download_full() || ph->error_download_thumbnail())) { + // Add photo to the unquarantined map. We add the photo + // to a map for later processing as there may be more + // episodes which referenced the same photo before it + // was quarantined. + unquarantined[photo_ids[i]] = ph; + + // Re-post photo to episode so it can be queried. If the photo was + // actually removed before being quarantined, it will have the + // REMOVED label and will be re-removed when the episode is queried. + eh->Lock(); + eh->AddPhoto(photo_ids[i]); + eh->SaveAndUnlock(updates); + + // Set episode invalidation so we re-query photos, and where not + // manually removed (no REMOVED label will arrive with photo), + // will re-post the photo to the episode in question. Assuming the + // download of missing image assets is successful, the photo will + // be correctly restored. + if (!episode_invalidated && !eh->upload_episode()) { + LOG("db migration: invalidating episode %s", eh->id()); + EpisodeSelection s; + s.set_episode_id(eh->id().server_id()); + s.set_get_photos(true); + state_->episode_table()->Invalidate(s, updates); + episode_invalidated = true; + } + } + } + } + } + + // Clear error label on all unquarantined photos so we proceed with + // another download attempt. + for (std::unordered_map::iterator iter = + unquarantined.begin(); + iter != unquarantined.end(); + ++iter) { + PhotoHandle ph = iter->second; + LOG("db migration: resetting error bit on photo %s", *ph); + ph->Lock(); + ph->clear_label_error(); + if (ph->error_download_thumbnail()) { + ph->set_download_thumbnail(true); + } + if (ph->error_download_full()) { + ph->set_download_full(true); + } + ph->SaveAndUnlock(updates); + } +} + +void DBMigration::MoveAssetKeys(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::photo_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + PhotoMetadata photo; + if (!photo.ParseFromString(ToString(value))) { + continue; + } + + if (photo.id().has_deprecated_asset_key()) { + const string& asset_key = photo.id().deprecated_asset_key(); + bool found = false; + for (int i = 0; i < photo.asset_keys_size(); i++) { + if (photo.asset_keys(i) == asset_key) { + found = true; + break; + } + } + if (!found) { + photo.add_asset_keys(asset_key); + } + photo.mutable_id()->clear_deprecated_asset_key(); + CHECK(updates->PutProto(key, photo)); + } + } +} + +void DBMigration::MultipleEpisodeActivityIndex(const DBHandle& updates) { + // Delete existing episode-activity index. + for (DB::PrefixIterator iter(updates, DBFormat::episode_activity_key("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + // Recreate by re-saving every share activity. + for (DB::PrefixIterator iter(updates, DBFormat::activity_key()); + iter.Valid(); + iter.Next()) { + const int64_t activity_id = state_->activity_table()->DecodeContentKey(iter.key()); + ActivityHandle ah = state_->activity_table()->LoadActivity(activity_id, updates); + if (ah->has_share_new() || ah->has_share_existing()) { + ah->Lock(); + ah->SaveAndUnlock(updates); + } + } +} + +void DBMigration::RemoveDaySummaryRows(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::day_summary_row_key("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } +} + +void DBMigration::RemoveInvalidPlacemarks(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::placemark_key("")); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + Location l; + bool should_delete = + (!DecodePlacemarkKey(key, &l) || !PlacemarkTable::IsLocationValid(l)); + if (!should_delete) { + // Check the stored placemark. + Placemark pm; + should_delete = + (!updates->GetProto(key, &pm) || !PlacemarkTable::IsPlacemarkValid(pm)); + } + if (should_delete) { + updates->Delete(key); + } + } +} + +void DBMigration::UploadSavePhotos(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::activity_key()); + iter.Valid(); + iter.Next()) { + const int64_t activity_id = state_->activity_table()->DecodeContentKey(iter.key()); + ActivityHandle ah = state_->activity_table()->LoadActivity(activity_id, updates); + if (ah->has_save_photos()) { + ah->Lock(); + ah->set_upload_activity(true); + ah->SaveAndUnlock(updates); + } + } +} + +void DBMigration::CommentActivityIndex(const DBHandle& updates) { + // Recreate by re-saving every share activity. + for (DB::PrefixIterator iter(updates, DBFormat::activity_key()); + iter.Valid(); + iter.Next()) { + const int64_t activity_id = state_->activity_table()->DecodeContentKey(iter.key()); + ActivityHandle ah = state_->activity_table()->LoadActivity(activity_id, updates); + if (ah->has_post_comment()) { + ah->Lock(); + ah->SaveAndUnlock(updates); + } + } +} + +void DBMigration::RequeryUsers(const DBHandle& updates) { + // Requery all user metadata. + for (DB::PrefixIterator iter(updates, DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + int64_t user_id = 0; + if (DecodeUserIdKey(iter.key(), &user_id)) { + state_->contact_manager()->MaybeQueueUser(user_id, updates); + } + } +} + +void DBMigration::RequeryContacts(const DBHandle& updates) { + if (state_->contact_manager()->count() > 0) { + // Only requery contacts if we have contacts. + state_->contact_manager()->InvalidateAll(updates); + + // Clear out the existing contact data. We're going to query all of the + // contacts and users again and want to start from scratch. + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_contact_id_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_contact_name_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::full_text_index_key(ContactManager::kContactIndexName)); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + // Re-query every user-id referenced in any activity. + for (DB::PrefixIterator iter(updates, DBFormat::activity_key()); + iter.Valid(); + iter.Next()) { + ActivityMetadata m; + if (!m.ParseFromString(ToString(iter.value()))) { + continue; + } + state_->contact_manager()->MaybeQueueUser(m.user_id(), updates); + + typedef ::google::protobuf::RepeatedPtrField ContactArray; + const ContactArray* contacts = ActivityTable::GetActivityContacts(m); + if (contacts) { + for (int i = 0; i < contacts->size(); ++i) { + state_->contact_manager()->MaybeQueueUser( + contacts->Get(i).user_id(), updates); + } + } + } + } +} + +void DBMigration::SplitAssetKeys(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::photo_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + PhotoMetadata photo; + if (!photo.ParseFromArray(value.data(), value.size())) { + continue; + } + + bool changed = false; + StringSet fingerprints; + // Should initially be empty, but check anyway to be idempotent. + for (int i = 0; i < photo.asset_fingerprints_size(); i++) { + fingerprints.insert(photo.asset_fingerprints(i)); + } + + StringSet keys; + for (int i = 0; i < photo.asset_keys_size(); i++) { + Slice url, fingerprint; + if (!DecodeAssetKey(photo.asset_keys(i), &url, &fingerprint)) { + continue; + } + if (url.empty()) { + changed = true; + } else { + keys.insert(photo.asset_keys(i)); + } + if (!fingerprint.empty()) { + bool inserted = fingerprints.insert(ToString(fingerprint)).second; + changed = changed || inserted; + } + } + + if (changed) { + photo.clear_asset_keys(); + for (StringSet::iterator it = keys.begin(); it != keys.end(); ++it) { + photo.add_asset_keys(*it); + } + + photo.clear_asset_fingerprints(); + for (StringSet::iterator it = fingerprints.begin(); it != fingerprints.end(); ++it) { + photo.add_asset_fingerprints(*it); + } + + CHECK(updates->PutProto(key, photo)); + } + } +} + +void DBMigration::AssetFingerprintIndex(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_asset_reverse_key("")); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + Slice fingerprint; + if (!DecodeDeprecatedAssetReverseKey(key, &fingerprint, NULL)) { + continue; + } + + updates->Put(EncodeAssetFingerprintKey(fingerprint), value); + updates->Delete(key); + } +} + +static void RewriteContactIndexedNames(const Slice& key, const Slice& value, const DBHandle& updates) { + ContactMetadata m; + if (!m.ParseFromArray(value.data(), value.size())) { + return; + } + if (m.indexed_names_size() > 0) { + for (int i = 0; i < m.indexed_names_size(); i++) { + m.set_indexed_names(i, DBFormat::deprecated_contact_name_key() + + (string)Format("%s %s", m.indexed_names(i), m.primary_identity())); + } + CHECK(updates->PutProto(key, m)); + } +} + +void DBMigration::ContactIndexStoreRaw(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + RewriteContactIndexedNames(iter.key(), iter.value(), updates); + } +} + +void DBMigration::ContactIdIndexStoreRaw(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_contact_id_key()); + iter.Valid(); + iter.Next()) { + RewriteContactIndexedNames(iter.key(), iter.value(), updates); + } +} + +void DBMigration::SetCoverPhotos(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get()) { + continue; + } + + PhotoHandle ph; + EpisodeHandle eh = vh->GetAnchorEpisode(&ph); + if (ph.get()) { + DCHECK(eh.get()); + if (!eh.get()) { + continue; + } + vh->Lock(); + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom(ph->id()); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom(eh->id()); + vh->SaveAndUnlock(updates); + } + } +} + +void DBMigration::QuarantinedPhotoEpisode(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::episode_photo_key("")); + iter.Valid(); + iter.Next()) { + if (iter.value() != EpisodeTable::kQuarantinedValue) { + continue; + } + + int64_t photo_id; + int64_t episode_id; + if (DecodeEpisodePhotoKey(iter.key(), &episode_id, &photo_id)) { + const string pe_key = EncodePhotoEpisodeKey(photo_id, episode_id); + if (!updates->Exists(pe_key)) { + LOG("restoring photo episode key for %d => %d", photo_id, episode_id); + updates->Put(pe_key, string()); + } + } + } +} + +void DBMigration::SaveComingledLibrary(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get() || vh->is_default()) { + continue; + } + // Super-special case for Spencer, Brett, Andy & Mike to avoid comingling. + if (state_->user_id() == 1 || + state_->user_id() == 6 || + state_->user_id() == 11 || + state_->user_id() == 89) { + if (!vh->deprecated_label_personal()) { + continue; + } + } + + // Get list of episodes in the viewpoint. + vector episodes; + vh->ListEpisodes(&episodes); + + // Compile the vec of photo_id/episode_id pairs for saving. + std::unordered_set unique_photo_ids; + PhotoSelectionVec photo_ids; + for (int i = 0; i < episodes.size(); ++i) { + // Don't save from episodes which were shared by this user. + if (episodes[i]->user_id() == state_->user_id()) { + continue; + } + vector ep_photo_ids; + episodes[i]->ListPhotos(&ep_photo_ids); + for (int j = 0; j < ep_photo_ids.size(); ++j) { + const int64_t photo_id = ep_photo_ids[j]; + if (!ContainsKey(unique_photo_ids, photo_id) && + !state_->photo_table()->PhotoInLibrary(photo_id, updates)) { + photo_ids.push_back(PhotoSelection(photo_id, episodes[i]->id().local_id())); + } + unique_photo_ids.insert(photo_id); + } + } + + // Save photos to library. + if (!photo_ids.empty()) { + state_->viewpoint_table()->SavePhotos(photo_ids, 0 /* autosave_viewpoint_id */, updates); + } + + if (vh->deprecated_label_personal()) { + // Unset personal label on conversation. + vh->Lock(); + vh->clear_deprecated_label_personal(); + vh->set_update_follower_metadata(true); + vh->SaveAndUnlock(updates); + } + } +} + +void DBMigration::SplitContactsUsers(const DBHandle& updates) { + const WallTime now = WallTime_Now(); + // First, delete the old indexes. + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_contact_id_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::deprecated_contact_name_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + + // Next, read all the contacts into memory. + vector contacts; + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + // The old schema denormalized contacts, so discard any redundant copies. + if (iter.key() != DBFormat::contact_key(m.primary_identity())) { + continue; + } + contacts.push_back(m); + } + + // Finally, reconstruct the database. + for (int i = 0; i < contacts.size(); i++) { + const ContactMetadata& m = contacts[i]; + if (m.user_id()) { + // Transfer the appropriate fields to a new user object + ContactMetadata user; + // Strip out "viewfinder" identities (which are just user ids) + if (!m.primary_identity().empty() && !IdentityManager::IsViewfinderIdentity(m.primary_identity())) { + user.set_primary_identity(m.primary_identity()); + } + for (int j = 0; j < m.deprecated_identities_size(); j++) { + if (!IdentityManager::IsViewfinderIdentity(m.deprecated_identities(j)) && + !m.deprecated_identities(j).empty()) { + user.add_identities()->set_identity(m.deprecated_identities(j)); + } + } + if (!m.name().empty()) { + user.set_name(m.name()); + } + if (!m.first_name().empty()) { + user.set_first_name(m.first_name()); + } + if (!m.last_name().empty()) { + user.set_last_name(m.last_name()); + } + if (!m.nickname().empty()) { + user.set_nickname(m.nickname()); + } + user.set_user_id(m.user_id()); + if (m.has_merged_with()) { + user.set_merged_with(m.merged_with()); + } + if (!m.email().empty()) { + user.set_email(m.email()); + } + if (m.has_label_registered()) { + user.set_label_registered(m.label_registered()); + } + if (m.has_label_terminated()) { + user.set_label_terminated(m.label_terminated()); + } + state_->contact_manager()->SaveUser(user, now, updates); + } else { + // No user id, so it's just a contact. + // Contacts don't have VF: identities or rules about user_*name, so we can almost use the old proto directly + // (but it does need to be mutable); + ContactMetadata copy(m); + for (int j = 0; j < m.deprecated_identities_size(); j++) { + if (!IdentityManager::IsViewfinderIdentity(m.deprecated_identities(j)) && + !m.deprecated_identities(j).empty()) { + copy.add_identities()->set_identity(m.deprecated_identities(j)); + } + } + if (copy.identities_size() == 0) { + continue; + } + copy.clear_deprecated_identities(); + if (!copy.has_contact_source()) { + copy.set_contact_source(ContactManager::GetContactSourceForIdentity(copy.primary_identity())); + } + state_->contact_manager()->SaveContact(copy, false, now, updates); + } + } +} + +void DBMigration::ContactAliasCleanup(const DBHandle& updates) { + // Delete the records that should have been deleted by the previous migration. + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + if (iter.key().starts_with("c/Email:") || + iter.key().starts_with("c/FacebookGraph:") || + iter.key().starts_with("c/Phone:") || + iter.key().starts_with("c/VF:")) { + updates->Delete(iter.key()); + } + } +} + +void DBMigration::DeleteEmptyContact(const DBHandle& updates) { + const string key("c/"); + if (updates->Exists(key)) { + updates->Delete(key); + } +} + +void DBMigration::RequerySelf(const DBHandle& updates) { + if (state_->user_id()) { + state_->contact_manager()->QueueUser(state_->user_id(), updates); + } +} + +void DBMigration::IndexServerContactId(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + if (m.has_server_contact_id()) { + CHECK(m.has_contact_id()); + updates->Put(DBFormat::server_contact_id_key(m.server_contact_id()), m.contact_id()); + } + } +} + +void DBMigration::ResetNeedQueryUser(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + // If the record doesn't look complete, queue it up to re-query. + // In particular, resolve_contacts could give us 'name' with no first/last name, + // and the 'registered' label used to be missing from contacts that are not friends. + if (m.name().empty() || + m.first_name().empty() || + m.last_name().empty() || + !m.label_registered() || + !m.label_friend()) { + m.set_need_query_user(true); + updates->PutProto(iter.key(), m); + state_->contact_manager()->QueueUser(m.user_id(), updates); + } + } +} + +void DBMigration::InvalidateContacts(const DBHandle& updates) { + if (state_->contact_manager()->count() > 0) { + // Only requery contacts if we have contacts. + state_->contact_manager()->InvalidateAll(updates); + } +} + +void DBMigration::CleanupContactIdentities(const DBHandle& updates) { + const WallTime now = WallTime_Now(); + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); iter.Valid(); iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + + // If any identities are the empty string, clear them. + if (m.has_primary_identity() && m.primary_identity().empty()) { + m.clear_primary_identity(); + } + for (int i = m.identities_size() - 1; i >= 0; i--) { + if (m.identities(i).identity().empty()) { + m.mutable_identities()->SwapElements(i, m.identities_size() - 1); + m.mutable_identities()->RemoveLast(); + } + } + + // If there are no identities, delete it (identity-less contacts have only been created by bugs). + if (m.primary_identity().empty() && + m.identities_size() == 0) { + if (m.contact_id().empty()) { + DCHECK(false) << "empty contact id in contact: " << m; + continue; + } + state_->contact_manager()->RemoveContact(m.contact_id(), true, updates); + continue; + } + + bool changed = false; + if (m.primary_identity().empty()) { + // If there is no primary identity, set it. + ContactManager::ChoosePrimaryIdentity(&m); + changed = true; + } else { + // Ensure the primary identity exists in the identities list. + StringSet identities; + for (int i = 0; i < m.identities_size(); i++) { + identities.insert(m.identities(i).identity()); + } + if (!ContainsKey(identities, m.primary_identity())) { + m.add_identities()->set_identity(m.primary_identity()); + changed = true; + if (m.identities_size() > 1) { + // Put the newly-added identity first in the list. + m.mutable_identities()->SwapElements(0, m.identities_size() - 1); + } + } + } + + if (changed) { + // If any changes were made, delete the old contact and re-save. + if (!m.contact_id().empty()) { + state_->contact_manager()->RemoveContact(m.contact_id(), true, updates); + m.clear_contact_id(); + } + state_->contact_manager()->SaveContact(m, true, now, updates); + } + } +} + +void DBMigration::MoveRemovedPhotosToHidden(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const int64_t ep_id = state_->episode_table()->DecodeContentKey(iter.key()); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ep_id, updates); + if (eh->id().has_server_id() && !eh->InLibrary()) { + vector photo_ids; + vector hide_ids; + eh->ListAllPhotos(&photo_ids); + for (int i = 0; i < photo_ids.size(); ++i) { + // Consider only removed photos. + if (!eh->IsRemoved(photo_ids[i])) { + continue; + } + hide_ids.push_back(photo_ids[i]); + } + + // Hide photos in episode. + if (!hide_ids.empty()) { + eh->Lock(); + for (int i = 0; i < hide_ids.size(); ++i) { + eh->HidePhoto(hide_ids[i]); + } + eh->SaveAndUnlock(updates); + } + } + } +} + +void DBMigration::BuildFollowerTables(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get()) { + continue; + } + vh->Lock(); + + vector follower_ids; + vh->ListFollowers(&follower_ids); + std::unordered_set follower_set(follower_ids.begin(), follower_ids.end()); + + vector participants; + ListParticipants(state_, vh, &participants, updates); + for (int i = 0; i < participants.size(); ++i) { + if (participants[i].has_user_id()) { + const int64_t user_id = participants[i].user_id(); + vh->AddFollower(user_id); + follower_set.erase(user_id); + } + } + if (!follower_set.empty()) { + for (std::unordered_set::iterator iter = follower_set.begin(); + iter != follower_set.end(); + ++iter) { + vh->RemoveFollower(*iter); + } + } + + vh->SaveAndUnlock(updates); + } +} + +void DBMigration::CanonicalizeCommentViewpoint(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::comment_key()); + iter.Valid(); + iter.Next()) { + const int64_t c_id = state_->comment_table()->DecodeContentKey(iter.key()); + CommentHandle ch = state_->comment_table()->LoadComment(c_id, updates); + if (!ch.get()) { + continue; + } + ch->Lock(); + state_->viewpoint_table()->CanonicalizeViewpointId(ch->mutable_viewpoint_id(), updates); + ch->SaveAndUnlock(updates); + } +} + +void DBMigration::ReindexComments(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::comment_key()); + iter.Valid(); + iter.Next()) { + const int64_t c_id = state_->comment_table()->DecodeContentKey(iter.key()); + CommentHandle ch = state_->comment_table()->LoadComment(c_id, updates); + if (!ch.get()) { + continue; + } + ch->Lock(); + ch->SaveAndUnlock(updates); + + MaybeFlush(updates); + } +} + +void DBMigration::BuildFollowerGroups(const DBHandle& updates) { + state_->people_rank()->Reset(); + for (DB::PrefixIterator iter(updates, DBFormat::follower_group_key_deprecated("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::follower_group_key("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get()) { + continue; + } + + vector follower_ids; + vh->ListFollowers(&follower_ids); + if (!vh->label_removed()) { + state_->people_rank()->AddViewpoint(vh->id().local_id(), follower_ids, updates); + } + } +} + +void DBMigration::ReindexEpisodes(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const int64_t e_id = state_->episode_table()->DecodeContentKey(iter.key()); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(e_id, updates); + if (!eh.get()) { + continue; + } + eh->Lock(); + eh->SaveAndUnlock(updates); + + MaybeFlush(updates); + } +} + +void DBMigration::ReindexViewpoints(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get()) { + continue; + } + vh->Lock(); + vh->SaveAndUnlock(updates); + + MaybeFlush(updates); + } +} + +void DBMigration::DeleteIdlessContacts(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + + if (m.contact_id().empty()) { + // All contacts without ids should have been deleted by the combination of SplitContactsUsers, + // ContactAliasCleanup, and DeleteEmptyContact. However, we have seen these contacts on several + // beta users' devices, and they cause a crash in ReindexContacts. + LOG("deleting idless contact: %s", m); + updates->Delete(iter.key()); + } + } +} + +void DBMigration::ReindexContacts(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + + state_->contact_manager()->ReindexContact(&m, updates); + + MaybeFlush(updates); + } +} + +void DBMigration::ReindexUsers(const DBHandle& updates) { + const WallTime now = WallTime_Now(); + for (DB::PrefixIterator iter(updates, DBFormat::user_id_key()); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + if (!m.ParseFromArray(iter.value().data(), iter.value().size())) { + continue; + } + + state_->contact_manager()->SaveUser(m, now, updates); + + MaybeFlush(updates); + } +} + +// Remove terminated users as followers from their viewpoints. +void DBMigration::RemoveTerminatedFollowers(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::contact_key("")); + iter.Valid(); + iter.Next()) { + ContactMetadata m; + // Skip if we can't decode or (more likely) if the user isn't terminated. + if (!m.ParseFromArray(iter.value().data(), iter.value().size()) || + !m.has_user_id() || !m.label_terminated()) { + continue; + } + + vector viewpoint_ids; + state_->viewpoint_table()->ListViewpointsForUserId( + m.user_id(), &viewpoint_ids, updates); + + for (int i = 0; i < viewpoint_ids.size(); ++i) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + viewpoint_ids[i], updates); + if (!vh.get()) { + continue; + } + vh->Lock(); + vh->RemoveFollower(m.user_id()); + vh->SaveAndUnlock(updates); + } + + MaybeFlush(updates); + } +} + +// Remove all feed-event-related data from DB. +void DBMigration::RemoveFeedEventData(const DBHandle& updates) { + const string kFeedDayKeyPrefix = "fday/"; + const DBRegisterKeyIntrospect kFeedDayKeyIntrospect( + kFeedDayKeyPrefix, + [](Slice key) { return string(); }, + [](Slice value) { return string(); }); + for (DB::PrefixIterator iter(updates, kFeedDayKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + const string kDayFeedEventKeyPrefix = "dfev/"; + const DBRegisterKeyIntrospect kDayFeedEventKeyIntrospect( + kDayFeedEventKeyPrefix, + [](Slice key) { return string(); }, + [](Slice value) { return string(); }); + for (DB::PrefixIterator iter(updates, kDayFeedEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + const string kDayActivityInvalidationKeyPrefix = "dais/"; + const DBRegisterKeyIntrospect kDayActivityInvalidationKeyIntrospect( + kDayActivityInvalidationKeyPrefix, + [](Slice key) { return string(); }, + [](Slice value) { return string(); }); + for (DB::PrefixIterator iter(updates, kDayActivityInvalidationKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + const string kTrapdoorFeedEventKeyPrefix = "tfe/"; + const DBRegisterKeyIntrospect kTrapdoorFeedEventKeyIntrospect( + kTrapdoorFeedEventKeyPrefix, + [](Slice key) { return string(); }, + [](Slice value) { return string(); }); + for (DB::PrefixIterator iter(updates, kTrapdoorFeedEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + const string kEpisodeFeedEventKeyPrefix = "efes/"; + const DBRegisterKeyIntrospect kEpisodeFeedEventKeyIntrospect( + kEpisodeFeedEventKeyPrefix, + [](Slice key) { return string(); }, + [](Slice value) { return string(); }); + for (DB::PrefixIterator iter(updates, kEpisodeFeedEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + const string kFullFeedEventSummaryKey = DBFormat::metadata_key("full_feed_event_summary"); + updates->Delete(kFullFeedEventSummaryKey); + + // Flush to ensure that all deletes go through using the + // still-registered introspection methods. + updates->Flush(false); +} + +void DBMigration::PrepareViewpointGCQueue(const DBHandle& updates) { + // First, delete any existing, spurious viewpoints queued for GC. + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_gc_key("")); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + + // Create a GC key for every removed viewpoint. If viewpoint is + // unrevivable (e.g. user removed themselves), schedule for immediate + // GC; otherwise, schedule with standard expiration. + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, updates); + if (!vh.get() || !vh->label_removed()) { + continue; + } + const WallTime expiration = vh->GetGCExpiration(); + updates->Put(EncodeViewpointGCKey(vp_id, expiration), string()); + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/DBMigration.h b/clients/shared/DBMigration.h new file mode 100644 index 0000000..029eb87 --- /dev/null +++ b/clients/shared/DBMigration.h @@ -0,0 +1,205 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_DB_MIGRATION_H +#define VIEWFINDER_DB_MIGRATION_H + +#import "AppState.h" +#import "DB.h" + +class DBMigration { + public: + DBMigration(AppState* state, ProgressUpdateBlock progress_update); + virtual ~DBMigration(); + + // Runs all migrations for the client which haven't yet been run. + // Returns true if any migration was performed. + bool MaybeMigrate(); + + // Switch from using a single migration version enum to keeping a + // row per migration step. This migration preserves accounting for + // migration steps up to and including MAYBE_UNQUARANTINE_PHOTOS. + void UseMigrationKeys(const DBHandle& updates); + + // Re-save every episode to generate the parent-child index. + void EpisodeParentChildIndex(const DBHandle& updates); + + // Set the "upload_metadata" bit for each episode if applicable. + void InitializeUploadEpisodeBit(const DBHandle& updates); + + // Check if photo has label_error() and either error_download_full + // or error_download_thumbnail, clear label_error() and requery the + // episode. + void MaybeUnquarantinePhotos(const DBHandle& updates); + + // Move the PhotoId.asset_key field to PhotoMetadata.asset_keys. + void MoveAssetKeys(const DBHandle& updates); + + // Rebuilds episode activity index to handle multiple activities + // sharing photos from within same episode. + void MultipleEpisodeActivityIndex(const DBHandle& updates); + + // Removes obsolete day summary row metadata. + void RemoveDaySummaryRows(const DBHandle& updates); + + // Removes placemarks with bad locations or invalid placemark data. + void RemoveInvalidPlacemarks(const DBHandle& updates); + + // Resets all pending save_photos activities to upload to the server. + void UploadSavePhotos(const DBHandle& updates); + + // Introduce secondary index mapping from comment server id to activity. + void CommentActivityIndex(const DBHandle& updates); + + // Requery all user metadata so that we can properly overlay user + // metadata on top of contact metadata. + void RequeryUsers(const DBHandle& updates); + + // Erase all contacts and requery. (see also InvalidateContacts below to + // requery without wiping the current state) + void RequeryContacts(const DBHandle& updates); + + // Splits asset_keys and asset_fingerprints fields; creates asset + // fingerprint reverse index and removes asset key reverse index. + void SplitAssetKeys(const DBHandle& updates); + + // Removes the reverse_asset_key index ("/ar/url#fingerprint") and + // creates the asset fingerprint index ("af/fingerprint"). + void AssetFingerprintIndex(const DBHandle& updates); + + // Rewrites the ContactMetadata::indexed_names field to include the + // exact data that was indexed. + void ContactIndexStoreRaw(const DBHandle& updates); + + // Same as ContactIndexStoreRaw, for contact_id_key instead of + // contact_key. + void ContactIdIndexStoreRaw(const DBHandle& updates); + + // Set cover photos for all viewpoints. + void SetCoverPhotos(const DBHandle& updates); + + // Restore photo=>episode link for instances where + // immediately-quarantined photos were skipped. + void QuarantinedPhotoEpisode(const DBHandle& updates); + + // Invoke "save_photos" api call on all episodes shared to + // conversations and include all photos which are not removed, + // unshared, or already present in the library. Unset the + // "personal" label if set on the conversation. + void SaveComingledLibrary(const DBHandle& updates); + + // Converts the contacts table to the new schema with contacts + // and users stored sepparately. + void SplitContactsUsers(const DBHandle& updates); + + // Deletes stale records left by a bug in SplitContactsUsers. + void ContactAliasCleanup(const DBHandle& updates); + + // Delete the erroneous empty contact record "c/" if it exists. + void DeleteEmptyContact(const DBHandle& updates); + + // Adds an index on the ContactMetadata::server_contact_id field. + void IndexServerContactId(const DBHandle& updates); + + // Requery the "self" user record in order to retrieve the user's identities + // and "no_password" setting. + void RequerySelf(const DBHandle& updates); + + // Sets the "need_query_user" flag on incomplete user records and queues them for querying. + void ResetNeedQueryUser(const DBHandle& updates); + + // Requeries all contacts and merges with the local state. + void InvalidateContacts(const DBHandle& updates); + + // Ensures that all contacts have an identity, the primary identity + // is also present in the identities array, and that the primary + // identity is set for all contacts + void CleanupContactIdentities(const DBHandle& updates); + + // Change the state of any "removed" photo to "hidden" if the + // episode is shared (e.g. not part of the default viewpoint). + void MoveRemovedPhotosToHidden(const DBHandle& updates); + + // Repair and verify the follower table as well as create the reverse + // follower table which looks up viewpoint ids by follower ids. + void BuildFollowerTables(const DBHandle& updates); + + // Add a local id to all comments' viewpoint_id field. + void CanonicalizeCommentViewpoint(const DBHandle& updates); + + // Re-saves all comments to update the full-text index. + void ReindexComments(const DBHandle& updates); + + // Build follower groups by iterating over all viewpoints and adding + // each to the follower group defined by the sorted array of + // follower ids. + void BuildFollowerGroups(const DBHandle& updates); + + // Re-saves all episodes to update the full-text index. + void ReindexEpisodes(const DBHandle& updates); + + // Re-saves all viewpoints to update the full-text index. + void ReindexViewpoints(const DBHandle& updates); + + // Deletes any contact records without a contact_id set. + void DeleteIdlessContacts(const DBHandle& updates); + + // Re-saves all contacts to update the full-text index. + void ReindexContacts(const DBHandle& updates); + + // Re-saves all users to update the full-text index. + void ReindexUsers(const DBHandle& updates); + + // Removes any terminated user ids from viewpoint followers. + void RemoveTerminatedFollowers(const DBHandle& updates); + + // Removes old feed event day table data from DB. + void RemoveFeedEventData(const DBHandle& updates); + + // Remove local-only photos in order to minimize the duplicate detection work + // required when upgrading to iOS 7. + virtual void RemoveLocalOnlyPhotos(const DBHandle& updates) { }; + + // Convert from old to new asset fingerprints. + virtual void ConvertAssetFingerprints(const DBHandle& updates) { }; + + // Index photos via their perceptual fingerprint. + virtual void IndexPhotos(const DBHandle& updates) { }; + + // Prepare the viewpoint garbage collection queue. + void PrepareViewpointGCQueue(const DBHandle& updates); + + // Remove photos with duplicate asset keys which have not been uploaded to + // the server. + virtual void RemoveAssetDuplicatePhotos(const DBHandle& updates) { } + + protected: + // Runs "migrator" and "progress_update" blocks if the migration + // hasn't yet been done. + typedef void (DBMigration::*migration_func)(const DBHandle&); + void RunMigration(const string& migration_key, migration_func migrator, + const DBHandle& updates); + + // Runs "migrator" and "progress_update" blocks if the migration hasn't been + // done yet, but only if running on iOS and "min_ios_version <= kIOSVersion < + // max_ios_version". Either or both of min_ios_version and max_ios_version + // can be NULL which unrestricts that range of the version check. + virtual void RunIOSMigration(const char* min_ios_version, const char* max_ios_version, + const string& migration_key, migration_func migrator, + const DBHandle& updates) = 0; + + // Flushes the transaction if it has a lot of pending data. + // Should only be used in idempotent migrations. + void MaybeFlush(const DBHandle& updates); + + private: + AppState* state_; + ProgressUpdateBlock progress_update_; + bool migrated_; +}; + +#endif // VIEWFINDER_DB_MIGRATION_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/DBStats.cc b/clients/shared/DBStats.cc new file mode 100644 index 0000000..47625aa --- /dev/null +++ b/clients/shared/DBStats.cc @@ -0,0 +1,158 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import "AppState.h" +#import "DayTable.h" +#import "DBStats.h" +#import "ViewpointTable.h" + +DBStats::DBStats(AppState* state) + : state_(state) { +} + +void DBStats::ComputeStats() { + ComputeViewpointStats(); + ComputeEventStats(); +} + +void DBStats::ComputeViewpointStats() { + int vp_count = 0; + int unique_count = 0; + std::unordered_set unique_sets; + std::map histogram; + + for (DB::PrefixIterator iter(state_->db(), DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, state_->db()); + vector follower_ids; + vh->ListFollowers(&follower_ids); + std::sort(follower_ids.begin(), follower_ids.end()); + string vec_str = ToString(follower_ids); + if (!follower_ids.empty() && !ContainsKey(unique_sets, vec_str)) { + unique_sets.insert(vec_str); + unique_count++; + histogram[follower_ids.size()]++; + } + vp_count++; + } + + LOG("%d unique sets of followers from %d viewpoints", unique_sets.size(), vp_count); + LOG("VIEWPOINT UNIQUE USER COUNTS:"); + int cumulative_count = 0; + for (std::map::iterator iter = histogram.begin(); + iter != histogram.end(); + ++iter) { + cumulative_count += iter->second; + LOG("%d followers: %d\t%0.2f%%ile", iter->first, iter->second, + float(cumulative_count * 100) / unique_count); + } +} + +void DBStats::ComputeEventStats() { + int ev_count = 0; + int ep_count = 0; + int ph_count = 0; + int tr_count = 0; + std::map event_photo_hist; + std::map event_episode_hist; + std::map event_trapdoor_hist; + std::map episode_photo_hist; + + for (DB::PrefixIterator iter(state_->db(), DBFormat::day_event_key("")); + iter.Valid(); + iter.Next()) { + const Slice value = iter.value(); + EventMetadata em; + if (em.ParseFromArray(value.data(), value.size())) { + if (em.episodes_size() > 0) { + int ev_ph_count = 0; + int ev_ep_count = 0; + for (int i = 0; i < em.episodes_size(); ++i) { + const FilteredEpisode& fe = em.episodes(i); + // Lots of derivative episodes which are filtered out because + // they're completely redundant. + if (fe.photo_ids_size() == 0) { + continue; + } + ph_count += fe.photo_ids_size(); + ev_ph_count += fe.photo_ids_size(); + ep_count++; + ev_ep_count++; + episode_photo_hist[fe.photo_ids_size()]++; + } + ev_count++; + event_photo_hist[ev_ph_count]++; + event_episode_hist[em.episodes_size()]++; + + // Trapdoors. + event_trapdoor_hist[em.trapdoors_size()]++; + tr_count += em.trapdoors_size(); + } + } + } + + LOG("%d events, %d episodes, %d photos, %d trapdoors", + ev_count, ep_count, ph_count, tr_count); + + if (!episode_photo_hist.empty()) { + LOG("EPISODE PHOTO COUNTS:"); + int cumulative_count = 0; + for (std::map::iterator iter = episode_photo_hist.begin(); + iter != episode_photo_hist.end(); + ++iter) { + cumulative_count += iter->second; + LOG("%d photos: %d\t%0.2f%%ile", iter->first, iter->second, + float(cumulative_count * 100) / ep_count); + } + } + + if (!event_episode_hist.empty()) { + LOG("EVENT EPISODE COUNTS:"); + int cumulative_count = 0; + for (std::map::iterator iter = event_episode_hist.begin(); + iter != event_episode_hist.end(); + ++iter) { + cumulative_count += iter->second; + LOG("%d episodes: %d\t%0.2f%%ile", iter->first, iter->second, + float(cumulative_count * 100) / ev_count); + } + } + + if (!event_photo_hist.empty()) { + LOG("EVENT PHOTO COUNTS:"); + int cumulative_count = 0; + for (std::map::iterator iter = event_photo_hist.begin(); + iter != event_photo_hist.end(); + ++iter) { + cumulative_count += iter->second; + LOG("%d photos: %d\t%0.2f%%ile", iter->first, iter->second, + float(cumulative_count * 100) / ev_count); + } + } + + if (!event_trapdoor_hist.empty()) { + LOG("EVENT TRAPDOOR COUNTS:"); + int cumulative_count = 0; + for (std::map::iterator iter = event_trapdoor_hist.begin(); + iter != event_trapdoor_hist.end(); + ++iter) { + cumulative_count += iter->second; + LOG("%d trapdoors: %d\t%0.2f%%ile", iter->first, iter->second, + float(cumulative_count * 100) / ev_count); + } + } + + int cumulative_count = 0; + float last = 0; + for (int i = 0; i < 20; ++i) { + if (ContainsKey(event_trapdoor_hist, i)) { + cumulative_count += event_trapdoor_hist[i]; + LOG("%0.2f", float(cumulative_count * 100) / ev_count); + last = float(cumulative_count * 100) / ev_count; + } else { + LOG("%0.2f", last); + } + } +} diff --git a/clients/shared/DBStats.h b/clients/shared/DBStats.h new file mode 100644 index 0000000..5c17f9c --- /dev/null +++ b/clients/shared/DBStats.h @@ -0,0 +1,31 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_DB_STATS_H +#define VIEWFINDER_DB_STATS_H + +#import "DB.h" + +class AppState; + +class DBStats { + public: + DBStats(AppState* state); + ~DBStats() {} + + // Compute statistics over database information. + void ComputeStats(); + + private: + void ComputeViewpointStats(); + void ComputeEventStats(); + + private: + AppState* state_; +}; + +#endif // VIEWFINDER_DB_STATS_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/DayMetadata.proto b/clients/shared/DayMetadata.proto new file mode 100644 index 0000000..aa3ce16 --- /dev/null +++ b/clients/shared/DayMetadata.proto @@ -0,0 +1,276 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +import "ContentIds.proto"; +import "Location.proto"; +import "Placemark.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "DayMetadataPB"; + +message DayContributor { + enum ContributorType { + UNVIEWED_CONTENT = 1; // unviewed content + VIEWED_CONTENT = 2; // already-viewed content + NO_CONTENT = 4; // just a follower; no content ever added + } + + optional int64 user_id = 1; + optional ContributorType type = 2; + // Identity is set only for contributors without user ids, i.e. prospective users after they are added + // to the conversation and before we have recieved their user id from the server. + optional string identity = 3; +} + +message DayPhoto { + optional int64 photo_id = 1; + optional int64 episode_id = 2; + optional double aspect_ratio = 3; + optional double timestamp = 4; +} + +// A [possibly] pared-down subset of an episode which +// lists only photo ids which are not already part of +// an earlier episode also occurring in this event. +message FilteredEpisode { + optional int64 episode_id = 1; + repeated int64 photo_ids = 2; +} + +message TrapdoorMetadata { + enum Type { + INBOX = 1; + EVENT = 2; + } + + // The viewpoint. + optional int64 viewpoint_id = 1; + // The type of trapdoor. + optional Type type = 2; + // Cover photo id. + optional DayPhoto cover_photo = 3; + // Initial activity timestamp. + optional double earliest_timestamp = 4; + // Latest activity timestamp. + optional double latest_timestamp = 5; + // The event index if this trapdoor is anchored to an + // event on the same day. + optional int32 event_index = 6; + // All contributors, sorted by most recent activity. + repeated DayContributor contributors = 7; + // Sampled photos. + repeated DayPhoto photos = 8; + // True if photos is a sampling of total. + optional bool sub_sampled = 9; + // Count of photos shared to viewpoint. + optional int32 photo_count = 10; + // Count of comments posted to viewpoint. + optional int32 comment_count = 11; + // Maximum update sequence of photos. + optional int32 new_photo_count = 12; + // Maximum updates sequence of comments. + optional int32 new_comment_count = 13; + // True if there are any unviewed photos, comments or updates. + optional bool unviewed_content = 14; + // True if there are any activities still pending upload. + optional bool pending_content = 15; + // True if the viewpoint is muted. + optional bool muted = 16; + // True if the viewpoint is autosaved. + optional bool autosave = 17; +} + +message EventMetadata { + // Earliest episode timestamp. + optional double earliest_timestamp = 1; + // Latest episode timestamp. + optional double latest_timestamp = 2; + // Total number of photos in all episodes. + optional int32 photo_count = 3; + // Sorted by timestamp of least recent photo in the episode. + repeated FilteredEpisode episodes = 4; + // Sorted by number of photos contributed. + repeated DayContributor contributors = 5; + // Canonical location matching centroid of all episodes. + optional Location location = 6; + // Canonical placemark matching location. + optional Placemark placemark = 7; + // Distance from location to closest "top" location. + optional double distance = 8; + // "EVENT" trapdoors into viewpoints. + repeated TrapdoorMetadata trapdoors = 9; + // Title if the event has trapdoors to viewpoints. + optional string title = 10; + optional string short_title = 11; +} + +// CachedEpisode contains de-normalized episode metadata such as the +// list of valid photo ids and location/placemark derived from +// available photos. These are used to efficiently recompute events +// without incurring the overhead of requerying episode data ad nauseum. +message CachedEpisode { + // Episode metadata. + optional EpisodeId id = 1; + optional EpisodeId parent_id = 2; + optional ViewpointId viewpoint_id = 3; + optional int64 user_id = 4; + optional double timestamp = 5; + // Derived from constituent photos. + optional Location location = 6; + optional Placemark placemark = 7; + optional double earliest_photo_timestamp = 8; + optional double latest_photo_timestamp = 9; + repeated DayPhoto photos = 10; + // Derived metadata. + optional bool in_library = 11; +} + +message DayMetadata { + optional double timestamp = 1; + // Episodes which occurred during this day. Used to compute + // events. + repeated CachedEpisode episodes = 2; +} + +message SummaryRow { + enum RowType { + EVENT = 1; // event summary + FULL_EVENT = 2; // fully-expanded event + EVENT_TRAPDOOR = 4; // convo trapdoor in events + TRAPDOOR = 5; // convo trapdoor in conversations + } + + optional RowType type = 1; + optional double timestamp = 2; + + // day_timestamp is the day this event/trapdoor can be found in. It is + // usually equal to CanonicalizeTimestamp(timestamp), but not necessarily, + // e.g. for events that span the 4:30am cutoff. + optional double day_timestamp = 3; + // Identifier used to locate asset summary row represents. In the case + // of an event, this is the index of the day's events; in the case of + // a trapdoor, this is the viewpoint id. + optional int64 identifier = 4; + + optional float height = 5; + optional bool unviewed = 6; + optional float position = 7; // position in the vertical scroll + optional float weight = 8; // normalized weight for crowding algorithm + + // Used for computing normalized weights. + optional int32 photo_count = 9; + optional int32 comment_count = 10; // Only applies to trapdoor types + optional int32 contributor_count = 11; + optional int32 share_count = 12; // Only applies to events + optional double distance = 13; // Only applies to events + + // For events only, this is the episode id of the first episode in + // the event. + optional int64 episode_id = 14; + + // HACK: used in EventSummaryView's search feature to track rows for expansion and photo selection. + // TODO(ben): refactor to use a real search result abstraction. + // This field is never persisted to disk so it should be safe to reclaim the tag number + // when it's gone. + optional int32 original_row_index = 20; +} + +// Statistics over entire day table. +message SummaryMetadata { + repeated SummaryRow rows = 1; + + optional int32 unviewed_count = 6; // Only applies to conversations + optional int32 photo_count = 8; + optional float total_height = 9; +} + +// Summary information for all activities in a viewpoint. +message ViewpointSummaryMetadata { + message Contributor { + optional int64 user_id = 1; + optional double update_seq = 2; + // Identity is set only for contributors without user_ids; see comment in DayContributor above. + optional string identity = 3; + } + + enum ActivityRowType { + HEADER = 1; + ACTIVITY = 2; + REPLY_ACTIVITY = 3; + PHOTOS = 4; + TITLE = 5; + UPDATE = 6; + FOOTER = 7; + } + + message ActivityRow { + message Photo { + optional int64 photo_id = 1; + optional int64 episode_id = 2; + optional int64 parent_episode_id = 3; + } + + optional int64 activity_id = 1; + // Contributors to activity. If share_new or add_followers, will + // include all added followers as well. + repeated int64 user_ids = 2; + // Identities of any added followers without user ids. + repeated string user_identities = 14; + // Timestamp of activity. + optional double timestamp = 3; + // Type of activity row. + optional ActivityRowType type = 4; + // Height of the activity row. + optional float height = 5; + // Position of the activity row in the full viewpoint. + optional float position = 6; + // Index of row, counting combined thread comments as a single row. + // This is used to alternate background row colors. + optional int32 row_count = 7; + // True if activity is unviewed. + optional int64 update_seq = 8; + // True if activity is pending. + optional bool pending = 9; + // Type of thread connecting this activity to previous or next activity. + optional int32 thread_type = 10; + // List of photos in this activity (applies to share_{new|existing}. + // This list is filtered based on preceeding activities which may + // already have included the same photo ids. There will be exactly + // one photo here in the case of REPLY_ACTIVITY. + repeated Photo photos = 11; + // True if this activity is a comment. + optional bool is_comment = 12; + // True if the activity is provisional (i.e. hasn't been cleared for send + // to the server). Necessary so that we can properly enter edit mode when a + // viewpoint with provisional activities is viewed. + // + // TODO(peter): Would it be better to just have a "provisional_activity_id" + // field in ViewpointSummaryMetadata. + optional bool is_provisional_hint = 13; + } + + // The viewpoint. + optional int64 viewpoint_id = 1; + // Contributors, sorted by most recent activity. + repeated Contributor contributors = 2; + // The cover photo. + optional DayPhoto cover_photo = 3; + // Activity rows. + repeated ActivityRow activities = 4; + // Initial activity timestamp. + optional double earliest_timestamp = 5; + // Latest activity timestamp. + optional double latest_timestamp = 6; + // Total count of photos in viewpoint. + optional int32 photo_count = 7; + // Total count of comments in viewpoint. + optional int32 comment_count = 8; + // Count of photos part of unviewed activities. + optional int32 new_photo_count = 9; + // Count of comments part of unviewed activities. + optional int32 new_comment_count = 10; + // True if the viewpoint is provisional. + optional bool provisional = 11; + // Scroll to this row when viewpoint is viewed. + optional float scroll_to_row = 12; +} diff --git a/clients/shared/DayTable.cc b/clients/shared/DayTable.cc new file mode 100644 index 0000000..29b6d05 --- /dev/null +++ b/clients/shared/DayTable.cc @@ -0,0 +1,3911 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import +#import "AppState.h" +#import "AsyncState.h" +#import "CommentTable.h" +#import "ContactManager.h" +#import "DayTable.h" +#import "Defines.h" +#import "FontSymbols.h" +#import "LocationUtils.h" +#import "PhotoTable.h" +#import "PlacemarkHistogram.h" +#import "STLUtils.h" +#import "StringUtils.h" +#import "Timer.h" + +namespace { + +// Key for invalidation trigger for use with transactions involving DayTable. +const string kDayTableCommitTrigger = "DayTableInvalidation"; + +// Maintains a sequence of invalidation ids. +const string kDayTableInvalidationSeqNoKey = + DBFormat::metadata_key("next_day_table_invalidation_seq_no"); + +// The timezone for which the current day table metadata is built. +const string kDayTableTimezoneKey = + DBFormat::metadata_key("day_table_timezone"); + +// Event/Conversations summaries. +const string kEpisodeSummaryKey = DBFormat::metadata_key("episode_summary"); +const string kEventSummaryKey = DBFormat::metadata_key("event_summary"); +const string kFullEventSummaryKey = DBFormat::metadata_key("full_event_summary"); +const string kConversationSummaryKey = DBFormat::metadata_key("conversation_summary"); +const string kUnviewedConversationSummaryKey = + DBFormat::metadata_key("unviewed_conversation_summary"); + +// Increment to clear all cached day table entries. +const int64_t kDayTableFormat = 85; +const string kDayTableFormatKey = DBFormat::metadata_key("day_table_format"); + +const string kDayKeyPrefix = DBFormat::day_key(""); +const string kDayEventKeyPrefix = DBFormat::day_event_key(""); +const string kDayEpisodeInvalidationKeyPrefix = DBFormat::day_episode_invalidation_key(""); +const string kEpisodeEventKeyPrefix = DBFormat::episode_event_key(""); +const string kTrapdoorEventKeyPrefix = DBFormat::trapdoor_event_key(); +const string kTrapdoorKeyPrefix = DBFormat::trapdoor_key(""); +const string kUserInvalidationKeyPrefix = DBFormat::user_invalidation_key(""); +const string kViewpointConversationKeyPrefix = DBFormat::viewpoint_conversation_key(""); +const string kViewpointInvalidationKeyPrefix = DBFormat::viewpoint_invalidation_key(""); + +const int kDayInSeconds = 60 * 60 * 24; +const int kMinRefreshCount = 20; // Minimum number of days to refresh in a cycle +const float kCommentThreadThreshold = 60 * 60; +const WallTime kMinTimestamp = 2 * kDayInSeconds; +const WallTime kMaxTimestamp = std::numeric_limits::max(); +const float kPracticalDayOffset = 4.5 * 60 * 60; // In seconds (04:30) +// Threshold number of photos in an event between +// using 2 "unit" rows in the summary view and 3 "unit" rows. +const int kEventPhotoThreshold = 5; +// Maximum sample sizes for trapdoor assets, by type. +const int kTrapdoorPhotoCount = 6; + +const DBRegisterKeyIntrospect kDayKeyIntrospect( + kDayKeyPrefix, + [](Slice key) { + WallTime timestamp; + if (!DecodeDayKey(key, ×tamp)) { + return string(); + } + return string(Format("%s", DBIntrospect::timestamp(timestamp))); + }, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kDayEventKeyIntrospect( + kDayEventKeyPrefix, + [](Slice key) { + WallTime timestamp; + int index; + if (!DecodeDayEventKey(key, ×tamp, &index)) { + return string(); + } + return string(Format("%s/%d", DBIntrospect::timestamp(timestamp), index)); + }, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kDayEpisodeInvalidateKeyIntrospect( + kDayEpisodeInvalidationKeyPrefix, + [](Slice key) { + WallTime timestamp; + int64_t episode_id; + if (!DecodeDayEpisodeInvalidationKey(key, ×tamp, &episode_id)) { + return string(); + } + return string(Format("%s/%d", DBIntrospect::timestamp(timestamp), episode_id)); + }, + [](Slice value) { + return value.ToString(); + }); + +// Need this for migration which removes the obsolete +// day summary rows. +const DBRegisterKeyIntrospect kDaySummaryRowKeyIntrospect( + DBFormat::day_summary_row_key(""), [](Slice key) { + return key.ToString(); + }, NULL); + +const DBRegisterKeyIntrospect kEventSummaryKeyIntrospect( + kEventSummaryKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kConversationSummaryKeyIntrospect( + kConversationSummaryKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kFullEventSummaryKeyIntrospect( + kFullEventSummaryKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kUnviewedConversationSummaryKeyIntrospect( + kUnviewedConversationSummaryKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kEpisodeEventKeyIntrospect( + kEpisodeEventKeyPrefix, [](Slice key) { + if (!key.starts_with(kEpisodeEventKeyPrefix)) { + return string(); + } + key.remove_prefix(kEpisodeEventKeyPrefix.size()); + const int64_t episode_id = OrderedCodeDecodeVarint64(&key); + return string(Format("%d", episode_id)); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointConversationKeyIntrospect( + kViewpointConversationKeyPrefix, [](Slice key) { + if (!key.starts_with(kViewpointConversationKeyPrefix)) { + return string(); + } + key.remove_prefix(kViewpointConversationKeyPrefix.size()); + const int64_t viewpoint_id = OrderedCodeDecodeVarint64(&key); + return string(Format("%d", viewpoint_id)); + }, NULL); + +const DBRegisterKeyIntrospect kTrapdoorEventKeyIntrospect( + kTrapdoorEventKeyPrefix, [](Slice key) { + if (!key.starts_with(kTrapdoorEventKeyPrefix)) { + return string(); + } + key.remove_prefix(kTrapdoorEventKeyPrefix.size()); + // TODO(ben): unmangle the ordered code in the event portion of the key. + return key.as_string();; + }, NULL); + +const DBRegisterKeyIntrospect kTrapdoorKeyIntrospect( + kTrapdoorKeyPrefix, + [](Slice key) { + int64_t viewpoint_id; + if (!DecodeTrapdoorKey(key, &viewpoint_id)) { + return string(); + } + return string(Format("%d", viewpoint_id)); + }, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kUserInvalidationKeyIntrospect( + DBFormat::user_invalidation_key(""), + [](Slice key) { + int64_t user_id; + if (!DecodeUserInvalidationKey(key, &user_id)) { + return string(); + } + return string(Format("%d", user_id)); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointInvalidationKeyIntrospect( + DBFormat::viewpoint_invalidation_key(""), + [](Slice key) { + WallTime timestamp; + int64_t viewpoint_id; + int64_t activity_id; + if (!DecodeViewpointInvalidationKey(key, ×tamp, &viewpoint_id, &activity_id)) { + return string(); + } + return string(Format("%s/%d/%d", DBIntrospect::timestamp(timestamp), + viewpoint_id, activity_id)); + }, + [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kViewpointSummaryKeyIntrospect( + DBFormat::viewpoint_summary_key(), NULL, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +// Returns true if the placemark has enough valid components to be formatted +// for display. +bool IsValidPlacemark(const Placemark& placemark) { + return (placemark.has_sublocality() || + placemark.has_locality() || + placemark.has_state()) && + placemark.has_country(); +} + +// Sets contact first name and full name if available. If neither is available, +// set the user id. Returns false iff the user has been terminated. +bool InitializeContributor( + AppState* state, DayContributor* contrib, int64_t user_id, const string& identity) { + ContactMetadata c; + if (user_id) { + if (state->contact_manager()->LookupUser(user_id, &c) && + c.label_terminated()) { + return false; + } + contrib->set_user_id(user_id); + } else { + contrib->set_identity(identity); + } + return true; +} + +typedef google::protobuf::RepeatedPtrField DayContributorArray; + +// Gets a vector of contributors. Uses first names if there are more than +// one contributor or if "shorten" is true. If a contributor has only a +// user_id, attempts to lookup the name using the contact manager. +void GetContributors(AppState* state, const DayContributorArray& contributors, + int contributor_mask, bool shorten, vector* contrib_vec) { + for (int i = 0; i < contributors.size(); ++i) { + if (contributor_mask && !(contributors.Get(i).type() & contributor_mask)) { + continue; + } + // Lookup names via contact manager by user id. + const int64_t user_id = contributors.Get(i).user_id(); + ContactMetadata cm; + if (user_id) { + if (!state->contact_manager()->LookupUser(user_id, &cm) || + user_id != cm.user_id() /* merged user */) { + continue; + } + } else { + cm.set_primary_identity(contributors.Get(i).identity()); + cm.add_identities()->set_identity(cm.primary_identity()); + } + const string first = state->contact_manager()->FirstName(cm, true); + const string name = state->contact_manager()->FullName(cm, true); + + if (shorten) { + if (user_id != state->user_id()) { + if (!first.empty()) { + contrib_vec->push_back(first); + break; + } else if (!name.empty()) { + contrib_vec->push_back(name); + break; + } + } + continue; + } else if (contributors.size() > 1) { + if (!first.empty()) { + contrib_vec->push_back(first); + continue; + } + } + if (!name.empty()) { + contrib_vec->push_back(name); + } else if (!first.empty()) { + contrib_vec->push_back(first); + } + } +} + +struct Holiday { + const WallTime timestamp; + const string title; +}; + +struct HolidaysByTimestamp { + bool operator()(const Holiday& a, WallTime ts) const { + return a.timestamp < ts; + } +}; + +// These values are sorted by the timestamp. Take care that they remain so. +// TODO(spencer): fetch these values from the server. +const Holiday kUSHolidays[] = { + {1009843200, "New Year's Day"}, + {1011571200, "Martin Luther King Jr.'s Day"}, + {1013644800, "Valentine's Day"}, + {1013990400, "President's Day"}, + {1016323200, "St. Patrick's Day"}, + {1022457600, "Memorial Day"}, + {1025740800, "Independence Day"}, + {1030924800, "Labor Day"}, + {1036022400, "Halloween"}, + {1038441600, "Thanksgiving Day"}, + {1040688000, "Christmas Eve"}, + {1040774400, "Christmas Day"}, + {1041292800, "New Year's Eve"}, + {1041379200, "New Year's Day"}, + {1043020800, "Martin Luther King Jr.'s Day"}, + {1045180800, "Valentine's Day"}, + {1045440000, "President's Day"}, + {1047859200, "St. Patrick's Day"}, + {1053907200, "Memorial Day"}, + {1057276800, "Independence Day"}, + {1062374400, "Labor Day"}, + {1067558400, "Halloween"}, + {1069891200, "Thanksgiving Day"}, + {1072224000, "Christmas Eve"}, + {1072310400, "Christmas Day"}, + {1072828800, "New Year's Eve"}, + {1072915200, "New Year's Day"}, + {1074470400, "Martin Luther King Jr.'s Day"}, + {1076716800, "Valentine's Day"}, + {1076889600, "President's Day"}, + {1079481600, "St. Patrick's Day"}, + {1085961600, "Memorial Day"}, + {1088899200, "Independence Day"}, + {1094428800, "Labor Day"}, + {1099180800, "Halloween"}, + {1101340800, "Thanksgiving Day"}, + {1103846400, "Christmas Eve"}, + {1103932800, "Christmas Day"}, + {1104451200, "New Year's Eve"}, + {1104537600, "New Year's Day"}, + {1105920000, "Martin Luther King Jr.'s Day"}, + {1108339200, "Valentine's Day"}, + {1108944000, "President's Day"}, + {1111017600, "St. Patrick's Day"}, + {1117411200, "Memorial Day"}, + {1120435200, "Independence Day"}, + {1125878400, "Labor Day"}, + {1130716800, "Halloween"}, + {1132790400, "Thanksgiving Day"}, + {1135382400, "Christmas Eve"}, + {1135468800, "Christmas Day"}, + {1135987200, "New Year's Eve"}, + {1136073600, "New Year's Day"}, + {1137369600, "Martin Luther King Jr.'s Day"}, + {1139875200, "Valentine's Day"}, + {1140393600, "President's Day"}, + {1142553600, "St. Patrick's Day"}, + {1148860800, "Memorial Day"}, + {1151971200, "Independence Day"}, + {1157328000, "Labor Day"}, + {1162252800, "Halloween"}, + {1164240000, "Thanksgiving Day"}, + {1166918400, "Christmas Eve"}, + {1167004800, "Christmas Day"}, + {1167523200, "New Year's Eve"}, + {1167609600, "New Year's Day"}, + {1168819200, "Martin Luther King Jr.'s Day"}, + {1171411200, "Valentine's Day"}, + {1171843200, "President's Day"}, + {1174089600, "St. Patrick's Day"}, + {1180310400, "Memorial Day"}, + {1183507200, "Independence Day"}, + {1188777600, "Labor Day"}, + {1193788800, "Halloween"}, + {1195689600, "Thanksgiving Day"}, + {1198454400, "Christmas Eve"}, + {1198540800, "Christmas Day"}, + {1199059200, "New Year's Eve"}, + {1199145600, "New Year's Day"}, + {1200873600, "Martin Luther King Jr.'s Day"}, + {1202947200, "Valentine's Day"}, + {1203292800, "President's Day"}, + {1205712000, "St. Patrick's Day"}, + {1211760000, "Memorial Day"}, + {1215129600, "Independence Day"}, + {1220227200, "Labor Day"}, + {1225411200, "Halloween"}, + {1227744000, "Thanksgiving Day"}, + {1230076800, "Christmas Eve"}, + {1230163200, "Christmas Day"}, + {1230681600, "New Year's Eve"}, + {1230768000, "New Year's Day"}, + {1232323200, "Martin Luther King Jr.'s Day"}, + {1234569600, "Valentine's Day"}, + {1234742400, "President's Day"}, + {1237248000, "St. Patrick's Day"}, + {1243209600, "Memorial Day"}, + {1246665600, "Independence Day"}, + {1252281600, "Labor Day"}, + {1256947200, "Halloween"}, + {1259193600, "Thanksgiving Day"}, + {1261612800, "Christmas Eve"}, + {1261699200, "Christmas Day"}, + {1262217600, "New Year's Eve"}, + {1262304000, "New Year's Day"}, + {1263772800, "Martin Luther King Jr.'s Day"}, + {1266105600, "Valentine's Day"}, + {1266192000, "President's Day"}, + {1268784000, "St. Patrick's Day"}, + {1275264000, "Memorial Day"}, + {1278201600, "Independence Day"}, + {1283731200, "Labor Day"}, + {1288483200, "Halloween"}, + {1290643200, "Thanksgiving Day"}, + {1293148800, "Christmas Eve"}, + {1293235200, "Christmas Day"}, + {1293753600, "New Year's Eve"}, + {1293840000, "New Year's Day"}, + {1295222400, "Martin Luther King Jr.'s Day"}, + {1297641600, "Valentine's Day"}, + {1298246400, "President's Day"}, + {1300320000, "St. Patrick's Day"}, + {1306713600, "Memorial Day"}, + {1309737600, "Independence Day"}, + {1315180800, "Labor Day"}, + {1320019200, "Halloween"}, + {1322092800, "Thanksgiving Day"}, + {1324684800, "Christmas Eve"}, + {1324771200, "Christmas Day"}, + {1325289600, "New Year's Eve"}, + {1325376000, "New Year's Day"}, + {1326672000, "Martin Luther King Jr.'s Day"}, + {1329177600, "Valentine's Day"}, + {1329696000, "President's Day"}, + {1331942400, "St. Patrick's Day"}, + {1338163200, "Memorial Day"}, + {1341360000, "Independence Day"}, + {1346630400, "Labor Day"}, + {1351641600, "Halloween"}, + {1353542400, "Thanksgiving Day"}, + {1356307200, "Christmas Eve"}, + {1356393600, "Christmas Day"}, + {1356912000, "New Year's Eve"}, +}; + +struct ActivityOlderThan { + bool operator()(const ActivityHandle& a, const ActivityHandle& b) const { + return a->timestamp() < b->timestamp(); + } +}; + +struct CachedEpisodeLessThan { + bool operator()(const CachedEpisode* a, const CachedEpisode* b) const { + if (a->earliest_photo_timestamp() != b->earliest_photo_timestamp()) { + return a->earliest_photo_timestamp() < b->earliest_photo_timestamp(); + } else if (b->has_parent_id() && + a->id().server_id() == b->parent_id().server_id()) { + return true; + } else if (a->in_library() != b->in_library()) { + return a->in_library(); + } else if (a->photos_size() != b->photos_size()) { + return a->photos_size() > b->photos_size(); + } else { + return a->id().local_id() < b->id().local_id(); + } + } +}; + +struct EpisodePhotoLessThan { + bool operator()(const std::pair& a, + const std::pair& b) const { + if (a.first != b.first) { + return a.first < b.first; + } + return a.second->timestamp() < b.second->timestamp(); + } +}; + +struct EventDistanceGreaterThan { + bool operator()(const DayTable::Event& a, + const DayTable::Event& b) const { + return a.distance() > b.distance(); + } +}; + +struct EventTimestampGreaterThan { + bool operator()(const DayTable::Event& a, + const DayTable::Event& b) const { + return a.latest_timestamp() > b.latest_timestamp(); + } +}; + +struct SummaryRowGreaterThan { + bool operator()(const SummaryRow* a, const SummaryRow* b) { + if (a->day_timestamp() != b->day_timestamp()) { + return a->day_timestamp() > b->day_timestamp(); + } + return a->identifier() < b->identifier(); + } + bool operator()(const SummaryRow* a, const SummaryRow& b) { + return (*this)(a, &b); + } + bool operator()(const SummaryRow& a, const SummaryRow& b) { + return (*this)(&a, &b); + } +}; + +struct TrapdoorGreaterThan { + bool operator()(const DayTable::Trapdoor& a, + const DayTable::Trapdoor& b) const { + return a.latest_timestamp() > b.latest_timestamp(); + } +}; + +struct ContributorNewerThan { + bool operator()(const ViewpointSummaryMetadata::Contributor& a, + const ViewpointSummaryMetadata::Contributor& b) const { + return a.update_seq() > b.update_seq(); + } +}; + + +// An iterator which merges activities and episodes into a sequence of +// days for which DayMetadata must be constructed. The day builder starts +// with the most recent day and iterates towards the least recent. +class DayBuilderIterator { + public: + DayBuilderIterator(AppState* state, const DBHandle& snapshot) + : done_(false), + activity_iter_(state->activity_table()->NewTimestampActivityIterator( + kMaxTimestamp, true, snapshot)), + episode_iter_(state->episode_table()->NewEpisodeIterator( + kMaxTimestamp, true, snapshot)), + cur_timestamp_(0) { + UpdateState(); + } + + // Advance the iterator. Sets done() to true if there are no more + // days in the iteration. + void Next() { + if (!done_) { + // NOTE: Even though cur_timestamp_ is a double, it is truncated + // to an int before being encoded for the index, so we can subtract + // 1 without fear of skipping any fractional entries. + activity_iter_->Seek(cur_timestamp_ - 1); + episode_iter_->Seek(cur_timestamp_ - 1); + UpdateState(); + } + } + + WallTime timestamp() const { return cur_timestamp_; } + bool done() const { return done_; } + + private: + void UpdateState() { + WallTime timestamp = 0; + if (activity_iter_->done() && episode_iter_->done()) { + done_ = true; + } else if (!activity_iter_->done() && !episode_iter_->done()) { + timestamp = CanonicalizeTimestamp( + std::max(activity_iter_->timestamp(), episode_iter_->timestamp())); + } else if (!activity_iter_->done()) { + timestamp = CanonicalizeTimestamp(activity_iter_->timestamp()); + } else if (!episode_iter_->done()) { + timestamp = CanonicalizeTimestamp(episode_iter_->timestamp()); + } + if (!done_) { + if (timestamp == cur_timestamp_) { + LOG("day table: day builder iterator done as timestamps equal %s, %d == %d", + WallTimeFormat("%b %e, %Y", timestamp), int(timestamp), int(cur_timestamp_)); + done_ = true; + } else { + cur_timestamp_ = timestamp; + } + } + } + + private: + bool done_; + ScopedPtr activity_iter_; + ScopedPtr episode_iter_; + // Invariant: cur_timestamp_ is always a "canonical" timestamp, aligned + // to day boundaries. + WallTime cur_timestamp_; +}; + +// Used to build and sort a list of event trapdoors. +struct TrapdoorProfile { + int photo_count; + int contrib_count; + string title; + TrapdoorProfile(int pc, int cc, const string& title) + : photo_count(pc), contrib_count(cc), title(title) {} +}; + +// Combines an existing stream of activities with a vector of +// ActivityHandle objects, sorted by timestamps. At each step of the +// iteration, ::should_append() can be consulted to determine whether +// the next activity should be taken from ::cur_activity() or built +// from scratch by calling ViewpointSummary::AppendActivityRows(). +typedef google::protobuf::RepeatedPtrField< + ViewpointSummaryMetadata::ActivityRow> ActivityRowArray; + +class ActivityMergeIterator { + public: + ActivityMergeIterator( + AppState* state, const DBHandle& db, const ActivityRowArray* existing, + const vector& ah_vec) + : state_(state), + db_(db), + existing_(existing), + existing_index_(0), + vec_index_(0), + cur_index_(-1), + prev_index_(-1), + next_index_(-1), + done_(false), + should_append_(false), + cur_is_existing_(true) { + for (int i = 0; i < ah_vec.size(); ++i) { + const bool already_contains_key = + ContainsKey(activity_ids_, ah_vec[i]->activity_id().local_id()); + DCHECK(!already_contains_key); + if (already_contains_key) { + continue; + } + activity_ids_.insert(ah_vec[i]->activity_id().local_id()); + // Verify activities are provided in sorted order. + if (i > 0) { + DCHECK_GE(ah_vec[i]->timestamp(), ah_vec[i - 1]->timestamp()); + } + // Remove non-visible activities. They're added to the activity + // ids set which will prevent them from being added from existing. + if (ah_vec[i]->IsVisible()) { + ah_vec_.push_back(ah_vec[i]); + } + } + // Skip existing, leading row(s) which are being replaced. + existing_index_ = NextExistingIndex(0, false); + // Set initial state. + UpdateInternal(); + } + + void Next() { + DCHECK(!done_); + if (done_) return; + if (cur_is_existing_) { + existing_index_ = NextExistingIndex(existing_index_ + 1, false); + DCHECK_LE(existing_index_, existing_->size()); + } else { + ++vec_index_; + DCHECK_LE(vec_index_, ah_vec_.size()); + } + UpdateInternal(); + } + + bool done() const { return done_; } + + const ActivityHandle& cur_ah() { + if (cur_index_ != -1 && !cur_ah_.get()) { + cur_ah_ = LoadActivity(cur_index_); + } + return cur_ah_; + } + const ActivityHandle& prev_ah() { + if (prev_index_ != -1 && !prev_ah_.get()) { + prev_ah_ = LoadActivity(prev_index_); + } + return prev_ah_; + } + const ActivityHandle& next_ah() { + if (next_index_ != -1 && !next_ah_.get()) { + next_ah_ = LoadActivity(next_index_); + } + return next_ah_; + } + + bool should_append() const { return should_append_; } + + const ViewpointSummaryMetadata::ActivityRow& cur_activity() const { + DCHECK_LT(existing_index_, existing_->size()); + return existing_->Get(existing_index_); + } + + private: + void UpdateInternal() { + const bool prev_is_existing = cur_is_existing_; + if (!IsExisting(existing_index_, vec_index_, &cur_is_existing_)) { + done_ = true; + return; + } + bool next_is_existing = false; + const int next_existing_index = + cur_is_existing_ ? NextExistingIndex(existing_index_ + 1, true) : existing_index_; + const int next_vec_index = cur_is_existing_ ? vec_index_ : vec_index_ + 1; + const bool has_next = IsExisting(next_existing_index, next_vec_index, &next_is_existing); + + // Always append the row fresh if it is an update + // (!cur_is_existing_), or either the previous or next is not + // existing. + should_append_ = !cur_is_existing_ || !prev_is_existing || !next_is_existing; + // If we're going to append, include the activity id in + // the activity ids set, so we skip all rows. + if (should_append_ && cur_is_existing_) { + activity_ids_.insert(existing_->Get(existing_index_).activity_id()); + } + + if (cur_ah_.get() || cur_index_ != -1) { + prev_ah_ = cur_ah_; + prev_index_ = cur_index_; + } else { + prev_ah_ = ActivityHandle(); + prev_index_ = -1; + } + if (cur_is_existing_) { + cur_ah_ = ActivityHandle(); + cur_index_ = existing_index_; + } else { + cur_ah_ = ah_vec_[vec_index_]; + cur_index_ = -1; + } + if (!has_next) { + next_ah_ = ActivityHandle(); + next_index_ = -1; + } else if (next_is_existing) { + next_ah_ = ActivityHandle(); + next_index_ = next_existing_index; + } else { + next_ah_ = ah_vec_[next_vec_index]; + next_index_ = -1; + } + } + + ActivityHandle LoadActivity(int index) { + const int64_t activity_id = existing_->Get(index).activity_id(); + return state_->activity_table()->LoadActivity(activity_id, db_); + } + + int NextExistingIndex(int index, bool skip_same_id) const { + while (index < existing_->size() && + (ContainsKey(activity_ids_, existing_->Get(index).activity_id()) || + (skip_same_id && index > 0 && + existing_->Get(index).activity_id() == existing_->Get(index - 1).activity_id()) || + existing_->Get(index).type() == ViewpointSummaryMetadata::HEADER)) { + ++index; + } + return index; + } + + bool IsExisting(int existing_index, int vec_index, bool* is_existing) const { + if (existing_index < existing_->size() && + vec_index < ah_vec_.size()) { + if (existing_->Get(existing_index).timestamp() < + ah_vec_[vec_index]->timestamp()) { + *is_existing = true; + } else { + *is_existing = false; + } + } else if (existing_index < existing_->size()) { + *is_existing = true; + } else if (vec_index < ah_vec_.size()) { + *is_existing = false; + } else { + // Case where the next is past the end of the iteration. + *is_existing = true; + return false; + } + return true; + } + + private: + AppState* const state_; + const DBHandle db_; + const ActivityRowArray* existing_; + vector ah_vec_; + std::unordered_set activity_ids_; + int existing_index_; + int vec_index_; + int64_t cur_index_; + int64_t prev_index_; + int64_t next_index_; + ActivityHandle cur_ah_; + ActivityHandle prev_ah_; + ActivityHandle next_ah_; + bool done_; + bool should_append_; + bool cur_is_existing_; +}; + +} // unnamed namespace + + +bool IsThreadTypeCombine(ActivityThreadType type) { + return (type == THREAD_COMBINE || + type == THREAD_COMBINE_NEW_USER || + type == THREAD_COMBINE_END || + type == THREAD_COMBINE_WITH_TIME || + type == THREAD_COMBINE_NEW_USER_WITH_TIME || + type == THREAD_COMBINE_END_WITH_TIME); +} + + +//// +// Event +// TODO(spencer): move this code below the Trapdoor object so it matches +// the header file. + +// Threshold between being near a "top" location (home) and far (away). +const double DayTable::Event::kHomeVsAwayThresholdMeters = 50 * 1000; // 50km + +// Geographic distance (in meters) before events are split when close +// to a "top" location. +const double DayTable::Event::kHomeThresholdMeters = 2.5 * 1000; // 2.5km +// Threshold for distance (in meters) when not close to a "top" location. +const double DayTable::Event::kAwayThresholdMeters = 10 * 1000; // 10km + +// Time in seconds before events are split when close to a "top" location. +const double DayTable::Event::kHomeThresholdSeconds = 4 * 60 * 60; // 4 hours +// Threshold for time (in seconds) when far from a "top" location. +const double DayTable::Event::kAwayThresholdSeconds = 6 * 60 * 60; // 6 hours + +// Time in seconds by which events can be extended past the trailing +// edge of the threshold to include episodes nearby in time. +const double DayTable::Event::kExtendThresholdRatio = 0.25; // 25% + +// Threshold distance in meters from top location to be considered "exotic". +const double DayTable::Event::kExoticThresholdMeters = 1000 * 1000; + + +DayTable::Event::Event(AppState* state, const DBHandle& db) + : state_(state), + db_(db) { +} + +string DayTable::Event::FormatTitle(bool shorten) const { + string exotic_suffix; + if (distance() >= kExoticThresholdMeters) { + exotic_suffix = kBoldSpaceSymbol + kBoldCompassSymbol; + } + + if (has_title()) { + // If we have a title from trapdoors we've contributed to, blend that with location. + string loc_str; + if (has_location() && has_placemark()) { + if (shorten) { + // Use just locality or sublocality for short location. + state_->placemark_histogram()->FormatLocality( + location(), placemark(), &loc_str); + return Format("%s%s%s%s%s", ToUppercase(loc_str), exotic_suffix, + kSpaceSymbol, kSpaceSymbol, short_title()); + } else { + // Otherwise, use shortened format of location. + state_->placemark_histogram()->FormatLocation( + location(), placemark(), true, &loc_str); + return Format("%s%s%s%s%s", ToUppercase(loc_str), exotic_suffix, + kSpaceSymbol, kSpaceSymbol, title()); + } + } else { + return shorten ? short_title() : title(); + } + } + + // Otherwise, use location if available, holiday, or default title. + return FormatLocation(shorten, true /* uppercase */); +} + +string DayTable::Event::FormatLocation(bool shorten, bool uppercase) const { + string title; + state_->day_table()->IsHoliday(CanonicalizeTimestamp(earliest_timestamp()), &title); + if (has_location() && has_placemark()) { + string loc_str; + state_->placemark_histogram()->FormatLocation( + location(), placemark(), shorten, &loc_str); + if (!title.empty()) { + return Format("%s%s%s%s", uppercase ? ToUppercase(loc_str) : loc_str, + kSpaceSymbol, kSpaceSymbol, title); + } else { + return Format("%s", uppercase ? ToUppercase(loc_str) : loc_str); + } + } else if (!title.empty()) { + return title; + } + return shorten ? "" : "Location Unavailable"; +} + +string DayTable::Event::FormatRelatedConvos(bool shorten) const { + if (!trapdoors_.size()) { + return ""; + } + string display_title; + if (has_title()) { + display_title = shorten ? short_title() : title(); + } else { + ViewpointHandle vh = + state_->viewpoint_table()->LoadViewpoint(trapdoors_[0]->viewpoint_id(), db_); + if (!vh.get()) { + return ""; + } + display_title = vh->FormatTitle(shorten, true); + } + if (trapdoors_.size() == 1 || shorten) { + return display_title; + } + return Format("%s and %d other%s", display_title, trapdoors_.size() - 1, + Pluralize(trapdoors_.size() - 1)); +} + +string DayTable::Event::FormatTimestamp(bool shorten) const { + return shorten ? + FormatShortRelativeDate(latest_timestamp(), state_->WallTime_Now()) : + FormatRelativeDate(latest_timestamp(), state_->WallTime_Now()); +} + +string DayTable::Event::FormatTimeRange(bool shorten) const { + return shorten ? + FormatRelativeTime(latest_timestamp(), state_->WallTime_Now()) : + ::FormatTimeRange(earliest_timestamp(), latest_timestamp()); +} + +string DayTable::Event::FormatContributors(bool shorten) const { + vector contrib_vec; + GetContributors(state_, contributors(), 0, shorten, &contrib_vec); + if (shorten) { + return contrib_vec.size() > 0 ? contrib_vec[0] : ""; + } else { + return Join(contrib_vec, ", "); + } +} + +string DayTable::Event::FormatPhotoCount() const { + return LocalizedNumberFormat(photo_count()); +} + +bool DayTable::Event::IsEmpty() const { + return photo_count() == 0; +} + +bool DayTable::Event::WithinTimeRange( + const CachedEpisode* anchor, WallTime timestamp, double margin_secs) { + const WallTime start_time = anchor->earliest_photo_timestamp() - margin_secs; + const WallTime end_time = anchor->latest_photo_timestamp() + margin_secs; + return timestamp >= start_time && timestamp <= end_time; +} + +bool DayTable::Event::ContainsPhotosFromEpisode(const CachedEpisode* ce) { + for (int i = 0; i < ce->photos_size(); ++i) { + if (ContainsKey(photo_ids_, ce->photos(i).photo_id())) { + return true; + } + } + return false; +} + +bool DayTable::Event::CanAddEpisode( + const CachedEpisode* anchor, const CachedEpisode* ce, float threshold_ratio) { + // Determine whether any photos in this episode are already in the + // anchor episode--that implies the episode and an episode in the event have + // a common ancestor. + if (ContainsPhotosFromEpisode(ce)) { + return true; + } + + // Is within the close home threshold of time? + const bool within_time_range = WithinTimeRange( + anchor, ce->timestamp(), kHomeThresholdSeconds * threshold_ratio); + + // TODO(spencer): should we factor in whether or not the episode + // is part of a viewpoint that is part of this event? + + // See if locations are similar enough to group. + if (ce->has_location() && anchor->has_location()) { + // Look at the distance as a function of how far anchor episode is + // from nearest "top" location. + double dist_to_top; + if (state_->placemark_histogram()->DistanceToLocation(anchor->location(), &dist_to_top)) { + const double dist_threshold = + threshold_ratio * ((dist_to_top < kHomeVsAwayThresholdMeters) ? + kHomeThresholdMeters : kAwayThresholdMeters); + const double time_threshold = + threshold_ratio * ((dist_to_top < kHomeVsAwayThresholdMeters) ? + kHomeThresholdSeconds : kAwayThresholdSeconds); + const bool within_time_range = WithinTimeRange(anchor, ce->timestamp(), time_threshold); + const double dist = DistanceBetweenLocations(anchor->location(), ce->location()); + if (dist < dist_threshold && within_time_range) { + return true; + } + } else { + // If for some reason the distance-to-location didn't work, base + // the decision off distance between locations compared to + // tightest, "home" threshold distance. + const float dist = DistanceBetweenLocations(anchor->location(), ce->location()); + const double dist_threshold = kHomeThresholdMeters * threshold_ratio; + if (dist < dist_threshold && within_time_range) { + return true; + } + } + } else if (within_time_range) { + // Without locations, fall back to whether the two episodes contain + // photos contributed by the same user(s). + return true; + } + + return false; +} + +void DayTable::Event::AddEpisode(const CachedEpisode* ce) { + // Keep track of earliest and latest timestamps. + if (!has_earliest_timestamp()) { + set_earliest_timestamp(std::numeric_limits::max()); + set_latest_timestamp(0); + } + set_earliest_timestamp(std::min(earliest_timestamp(), ce->earliest_photo_timestamp())); + set_latest_timestamp(std::max(latest_timestamp(), ce->latest_photo_timestamp())); + + episodes_.push_back(ce); + for (int i = 0; i < ce->photos_size(); ++i) { + photo_ids_.insert(ce->photos(i).photo_id()); + } +} + +const EventMetadata& DayTable::Event::Canonicalize() { + CHECK_GT(episodes_.size(), 0); + CanonicalizeEpisodes(); + CanonicalizeLocation(); + CanonicalizeTrapdoors(); + Cleanup(); + return *this; +} + +void DayTable::Event::Cleanup() { + episodes_.clear(); + photo_ids_.clear(); +} + +void DayTable::Event::CanonicalizeEpisodes() { + // Sort the episodes by earliest photo timestamp and number of photos. + std::sort(episodes_.begin(), episodes_.end(), CachedEpisodeLessThan()); + // Map from user id to contributed photo count. + std::unordered_map contributors; + + // Now, build vector of filtered episodes based on that ordering + // by eliminating duplicative photo ids. + std::unordered_set unique_photo_ids; + int count = 0; + for (int i = 0; i < episodes_.size(); ++i) { + const CachedEpisode* ce = episodes_[i]; + FilteredEpisode* f_ep = add_episodes(); + f_ep->set_episode_id(ce->id().local_id()); + // Don't include photos from episodes which aren't in library. + // We do want episode id however, as this is used to locate an + // episode from a conversation containing photos from the event. + if (!ce->in_library()) { + continue; + } + for (int j = 0; j < ce->photos_size(); ++j) { + if (!ContainsKey(unique_photo_ids, ce->photos(j).photo_id())) { + unique_photo_ids.insert(ce->photos(j).photo_id()); + f_ep->add_photo_ids(ce->photos(j).photo_id()); + // Add contributors, including current user. If current user is + // sole contributor, no contributors are added. + contributors[episodes_[i]->user_id()] += 1; + count += 1; + } + } + } + // Reset photo_count to account for duplicates. + set_photo_count(count); + + // Sort contributors by contributed photo counts. + vector > by_count; + for (std::unordered_map::iterator iter = contributors.begin(); + iter != contributors.end(); + ++iter) { + by_count.push_back(std::make_pair(iter->second, iter->first)); + } + // Clear contributors vector if it contains only the user. + if (by_count.size() == 1 && by_count.back().second == state_->user_id()) { + by_count.clear(); + } + + // Sort in order of most to least contributed photos. + std::sort(by_count.begin(), by_count.end(), std::greater >()); + for (int i = 0; i < by_count.size(); ++i) { + const int64_t user_id = by_count[i].second; + DayContributor contrib; + if (!InitializeContributor(state_, &contrib, user_id, "")) { + continue; + } + contrib.set_type(DayContributor::VIEWED_CONTENT); + contrib.Swap(add_contributors()); + } +} + +void DayTable::Event::CanonicalizeLocation() { + // Compute the location centroid and find an actual location nearest + // the centroid to choose its placemark. This helps to correct for + // spurious reverse geo-locations. + Location centroid; + int count = 0; + for (int i = 0; i < episodes_.size(); ++i) { + if (episodes_[i]->has_location() && + IsValidPlacemark(episodes_[i]->placemark())) { + const Location& location = episodes_[i]->location(); + centroid.set_latitude(centroid.latitude() + location.latitude()); + centroid.set_longitude(centroid.longitude() + location.longitude()); + centroid.set_accuracy(centroid.accuracy() + location.accuracy()); + centroid.set_altitude(centroid.altitude() + location.altitude()); + ++count; + } + } + + if (count == 0) { + return; + } + + centroid.set_latitude(centroid.latitude() / count); + centroid.set_longitude(centroid.longitude() / count); + centroid.set_accuracy(centroid.accuracy() / count); + centroid.set_altitude(centroid.altitude() / count); + + // Locate the episode closest to the centroid. + int closest_index = -1; + double closest_distance = std::numeric_limits::max(); + for (int i = 0; i < episodes_.size(); ++i) { + if (episodes_[i]->has_location() && + IsValidPlacemark(episodes_[i]->placemark())) { + const double distance = DistanceBetweenLocations( + centroid, episodes_[i]->location()); + if (distance < closest_distance) { + closest_distance = distance; + closest_index = i; + } + } + } + + // Use the location & placemark of the closest episode. + if (closest_index != -1) { + mutable_location()->CopyFrom(episodes_[closest_index]->location()); + mutable_placemark()->CopyFrom(episodes_[closest_index]->placemark()); + + // Handle case of pending reverse geocode. + // TODO(spencer): we really should be returning this more elegantly. + if (!IsValidPlacemark(placemark())) { + clear_placemark(); + } + double distance; + if (state_->placemark_histogram()->DistanceToLocation(location(), &distance)) { + set_distance(distance); + } + } +} + +void DayTable::Event::CanonicalizeTrapdoors() { + if (trapdoors_.empty()) { + return; + } + int contributor_count_max = 0; + int photo_count_max = 0; + vector profiles; + for (int i = 0; i < trapdoors_.size(); ++i) { + // Can only access Trapdoor::viewpoint_ before canonicalization. + const string title = trapdoors_[i]->viewpoint_->title(); + // Compute contributor count before canonicalization. + const int contrib_count = trapdoors_[i]->contributors_.size(); + // See if this trapdoor contains the anchor episode for the viewpoint... + EpisodeHandle anchor = trapdoors_[i]->viewpoint_->GetAnchorEpisode(NULL); + bool includes_anchor = false; + if (anchor.get()) { + for (int j = 0; j < trapdoors_[i]->episodes_.size(); ++j) { + const int64_t local_id = trapdoors_[i]->episodes_[j].first->id().local_id(); + if (local_id == anchor->id().local_id()) { + includes_anchor = true; + break; + } + } + } + + *add_trapdoors() = trapdoors_[i]->Canonicalize(); + + if (!title.empty() && includes_anchor) { + profiles.push_back( + TrapdoorProfile(trapdoors_[i]->photo_count(), contrib_count, title)); + photo_count_max = std::max(photo_count_max, + profiles.back().photo_count); + contributor_count_max = std::max(contributor_count_max, + profiles.back().contrib_count); + } + } + + if (!profiles.empty()) { + const float kContribWeight = 0.55; + const float kPhotosWeight = 0.45; + vector > weighted_indexes; + for (int i = 0; i < profiles.size(); ++i) { + const float norm_contrib = contributor_count_max ? + (profiles[i].contrib_count / contributor_count_max) : 0; + const float norm_photo = photo_count_max ? + (profiles[i].photo_count / photo_count_max) : 0; + const float weight = kContribWeight * norm_contrib + kPhotosWeight * norm_photo; + weighted_indexes.push_back(std::make_pair(weight, i)); + } + sort(weighted_indexes.begin(), weighted_indexes.end(), + std::greater >()); + const string& best_title = profiles[weighted_indexes.front().second].title; + set_title(NormalizeWhitespace(best_title)); + set_short_title(title()); + } +} + + +//// +// Trapdoor + +DayTable::Trapdoor::Trapdoor(AppState* state, const DBHandle& db) + : state_(state), + db_(db) { +} + +string DayTable::Trapdoor::FormatTimestamp(bool shorten) const { + return shorten ? + FormatShortRelativeDate(earliest_timestamp(), state_->WallTime_Now()) : + FormatRelativeDate(earliest_timestamp(), state_->WallTime_Now()); +} + +string DayTable::Trapdoor::FormatTimeAgo() const { + return ::FormatTimeAgo(latest_timestamp(), state_->WallTime_Now(), TIME_AGO_SHORT); +} + +string DayTable::Trapdoor::FormatContributors( + bool shorten, int contributor_mask) const { + vector contrib_vec; + GetContributors(state_, contributors(), contributor_mask, shorten, &contrib_vec); + if (shorten) { + return contrib_vec.size() > 0 ? contrib_vec[0] : ""; + } else { + return Join(contrib_vec, ", "); + } +} + +string DayTable::Trapdoor::FormatPhotoCount() const { + return LocalizedNumberFormat(photo_count()); +} + +string DayTable::Trapdoor::FormatCommentCount() const { + return LocalizedNumberFormat(comment_count()); +} + +bool DayTable::Trapdoor::DisplayInSummary() const { + // Display inbox trapdoors in the summary which have + // unviewed content. + if (type() == INBOX) { + return unviewed_content(); + } else { + return false; + } +} + +bool DayTable::Trapdoor::DisplayInInbox() const { + // Display all inbox trapdoors. + return type() == INBOX; +} + +bool DayTable::Trapdoor::DisplayInEvent() const { + // Display all event trapdoors in event view. You can access all + // EVENT trapdoors from the event view. + return type() == EVENT; +} + +bool DayTable::Trapdoor::IsEmpty() const { + return photo_count() == 0 && comment_count() == 0; +} + +ViewpointHandle DayTable::Trapdoor::GetViewpoint() const { + return state_->viewpoint_table()->LoadViewpoint(viewpoint_id(), db_); +} + +void DayTable::Trapdoor::InitFromViewpointSummary(const ViewpointSummaryMetadata& vs) { + viewpoint_ = state_->viewpoint_table()->LoadViewpoint(vs.viewpoint_id(), db_); + + set_viewpoint_id(viewpoint_->id().local_id()); + set_type(Trapdoor::INBOX); + + if (vs.has_cover_photo()) { + mutable_cover_photo()->CopyFrom(vs.cover_photo()); + } + set_earliest_timestamp(vs.earliest_timestamp()); + set_latest_timestamp(vs.latest_timestamp()); + + set_photo_count(vs.photo_count()); + set_comment_count(vs.comment_count()); + set_new_photo_count(vs.new_photo_count()); + set_new_comment_count(vs.new_comment_count()); + + // Determine unviewed / pending content flags and sample photos from + // most recent to least recent. + std::unordered_set unique_photo_ids; + for (int i = vs.activities_size() - 1; i >= 0; --i) { + const ViewpointSummaryMetadata::ActivityRow& row = vs.activities(i); + const bool unviewed = row.update_seq() > viewpoint_->viewed_seq(); + if (unviewed) { + set_unviewed_content(true); + } + // Sample photos if we have fewer than the trapdoor photo count. + if (row.type() == ViewpointSummaryMetadata::HEADER || + row.type() == ViewpointSummaryMetadata::PHOTOS) { + for (int j = row.photos_size() - 1; j >= 0; --j) { + const int64_t photo_id = row.photos(j).photo_id(); + if (ContainsKey(unique_photo_ids, photo_id)) { + continue; + } + unique_photo_ids.insert(photo_id); + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_id, db_); + if (!ph.get()) { + continue; + } + DayPhoto* photo = add_photos(); + photo->set_photo_id(ph->id().local_id()); + photo->set_episode_id(row.photos(j).episode_id()); + photo->set_aspect_ratio(ph->aspect_ratio()); + photo->set_timestamp(ph->timestamp()); + } + } + if (row.pending()) { + set_pending_content(true); + } + } + + if (viewpoint_->label_muted()) { + set_muted(true); + } + if (viewpoint_->label_autosave()) { + set_autosave(true); + } + + // Transfer contributors & canonicalize. We must do this after we've + // determined whether there's new content. + for (int i = 0; i < vs.contributors_size(); ++i) { + if (vs.contributors(i).user_id()) { + contributors_[vs.contributors(i).user_id()] = + vs.contributors(i).update_seq(); + } else { + contributors_by_identity_[vs.contributors(i).identity()] = + vs.contributors(i).update_seq(); + } + } + CanonicalizeContributors(); +} + +void DayTable::Trapdoor::AddSharedEpisode( + const ActivityHandle& ah, const CachedEpisode* ce) { + if (!ah->has_share_new() && !ah->has_share_existing()) { + return; + } + + if (!has_earliest_timestamp()) { + // Initialize first and last timestamps. + set_earliest_timestamp(ah->timestamp()); + set_latest_timestamp(ah->timestamp()); + } else { + CHECK_EQ(viewpoint_id(), ah->viewpoint_id().local_id()); + set_earliest_timestamp(std::min(earliest_timestamp(), ah->timestamp())); + set_latest_timestamp(std::max(latest_timestamp(), ah->timestamp())); + } + + if (ah->upload_activity()) { + set_pending_content(true); + } + + episodes_.push_back(std::make_pair(ce, false)); + + // Add sharees if available. + if (ah->has_share_new()) { + for (int i = 0; i < ah->share_new().contacts_size(); ++i) { + if (ah->share_new().contacts(i).has_user_id()) { + const int64_t user_id = ah->share_new().contacts(i).user_id(); + if (user_id) { + contributors_[user_id] = + std::max(contributors_[user_id], ah->update_seq()); + } else { + const string& identity = ah->share_new().contacts(i).primary_identity(); + contributors_by_identity_[identity] = + std::max(contributors_by_identity_[identity], ah->update_seq()); + } + } + } + } + + // Add contributor. + contributors_[ah->user_id()] = + std::max(contributors_[ah->user_id()], ah->update_seq() + 0.1); +} + +const TrapdoorMetadata& DayTable::Trapdoor::Canonicalize() { + CHECK(has_viewpoint_id()); + + SamplePhotos(); + MaybeSetCoverPhoto(); + CanonicalizeContributors(); + + Cleanup(); + return *this; +} + +void DayTable::Trapdoor::Cleanup() { + viewpoint_.reset(); + contributors_.clear(); + contributors_by_identity_.clear(); +} + +void DayTable::Trapdoor::SamplePhotos() { + // Go through all episodes and build a vector of episode + // photo id lists for episodes shared from an unviewed activity. + int available_count = 0; + for (int i = 0; i < episodes_.size(); ++i) { + const CachedEpisode* ce = episodes_[i].first; + const bool new_episode = episodes_[i].second; + + // Update photo counts only in the case of an EVENT type trapdoor. + if (type() == EVENT) { + if (new_episode) { + set_new_photo_count(new_photo_count() + ce->photos_size()); + } + set_photo_count(photo_count() + ce->photos_size()); + } + available_count += ce->photos_size(); + } + + // Sample photo ids using round-robin. + std::unordered_set sampled_ids; + int sample_count = std::min(available_count, kTrapdoorPhotoCount); + // Maintain vector as a pair of (episode rank, photo) for proper sorting. + vector > photos; + for (int round = 0, count = 0; count < sample_count; ++round) { + bool found = false; // safety check in case we can't meet sampling target + for (int i = 0; i < episodes_.size() && count < sample_count; ++i) { + const CachedEpisode* ce = episodes_[i].first; + if (round < ce->photos_size() && + !ContainsKey(sampled_ids, ce->photos(round).photo_id())) { + sampled_ids.insert(ce->photos(round).photo_id()); // prevent duplicates + photos.push_back(std::make_pair(i, &ce->photos(round))); + count++; + found = true; + } + } + if (!found) { + // We ran out of photos to sample because of duplicates. + break; + } + } + // Sort photos by timestamp in ascending order and copy to sample array. + std::sort(photos.begin(), photos.end(), EpisodePhotoLessThan()); + for (int i = 0; i < photos.size(); ++i) { + add_photos()->CopyFrom(*photos[i].second); + } + if (sample_count < available_count) { + set_sub_sampled(true); + } +} + +void DayTable::Trapdoor::MaybeSetCoverPhoto() { + int64_t cover_photo_id; + int64_t cover_episode_id; + WallTime cover_timestamp; + float cover_aspect_ratio; + if (viewpoint_->GetCoverPhoto(&cover_photo_id, + &cover_episode_id, + &cover_timestamp, + &cover_aspect_ratio)) { + mutable_cover_photo()->set_photo_id(cover_photo_id); + mutable_cover_photo()->set_episode_id(cover_episode_id); + mutable_cover_photo()->set_timestamp(cover_timestamp); + mutable_cover_photo()->set_aspect_ratio(cover_aspect_ratio); + } +} + +void DayTable::Trapdoor::CanonicalizeContributors() { + // The update sequence is a floating point value to allow the + // user adding additional users to sort first. + vector > > by_update_seq; + for (std::unordered_map::iterator iter = contributors_.begin(); + iter != contributors_.end(); + ++iter) { + // Add the user himself only if there is one contributor. + if (iter->first != state_->user_id() || + (contributors_.size() + contributors_by_identity_.size()) == 1) { + by_update_seq.push_back(std::make_pair(iter->second, std::make_pair(iter->first, ""))); + } + } + for (std::unordered_map::iterator iter = contributors_by_identity_.begin(); + iter != contributors_by_identity_.end(); + ++iter) { + by_update_seq.push_back(std::make_pair(iter->second, std::make_pair(0, iter->first))); + } + // Sort in order of most to least recent contributions. + std::sort(by_update_seq.begin(), by_update_seq.end(), + std::greater > >()); + for (int i = 0; i < by_update_seq.size(); ++i) { + const int max_update_seq = int(by_update_seq[i].first); + const int64_t user_id = by_update_seq[i].second.first; + const string& identity = by_update_seq[i].second.second; + DayContributor contrib; + if (!InitializeContributor(state_, &contrib, user_id, identity)) { + continue; + } + if (max_update_seq > viewpoint_->viewed_seq() && unviewed_content()) { + contrib.set_type(DayContributor::UNVIEWED_CONTENT); + } else { + contrib.set_type(DayContributor::VIEWED_CONTENT); + } + contrib.Swap(add_contributors()); + } + // Go through list of viewpoint followers and add any that haven't + // already been included in contributors. + vector follower_ids; + viewpoint_->ListFollowers(&follower_ids); + for (int i = 0; i < follower_ids.size(); ++i) { + if (!ContainsKey(contributors_, follower_ids[i])) { + DayContributor contrib; + if (!InitializeContributor(state_, &contrib, follower_ids[i], "")) { + continue; + } + contrib.set_type(DayContributor::NO_CONTENT); + contrib.Swap(add_contributors()); + } + } +} + + +//// +// Day + +DayTable::Day::Day(AppState* state, WallTime timestamp, const DBHandle& db) + : state_(state), + db_(db) { + CHECK_EQ(timestamp, CanonicalizeTimestamp(timestamp)); + metadata_.set_timestamp(timestamp); +} + +bool DayTable::Day::Load() { + // Get the day metadata from the snapshot database. + return db_->GetProto(EncodeDayKey(timestamp()), &metadata_); +} + +void DayTable::Day::Save( + vector* events, const DBHandle& updates) { + std::sort(events->begin(), events->end(), EventTimestampGreaterThan()); + for (int i = 0; i < events->size(); ++i) { + const EventMetadata& event = (*events)[i].Canonicalize(); + if ((*events)[i].IsEmpty()) { + // NOTE: this is O(N) and if a large number of events occur on a day, + // this code should be revamped for efficiency. + events->erase(events->begin() + i); + --i; + continue; + } + updates->PutProto(EncodeDayEventKey(timestamp(), i), event); + } + + updates->PutProto(EncodeDayKey(metadata_.timestamp()), metadata_); +} + +void DayTable::Day::Rebuild(vector* events, const DBHandle& updates) { + // Build a vector of all episodes which occurred on this day. + const WallTime end_timestamp = NextDay(timestamp()) + kPracticalDayOffset; + for (ScopedPtr iter( + state_->episode_table()->NewEpisodeIterator(timestamp(), false, db_)); + !iter->done() && iter->timestamp() < end_timestamp; + iter->Next()) { + EpisodeHandle eh = state_->episode_table()->LoadEpisode(iter->episode_id(), db_); + if (eh.get()) { + CachedEpisode* ce = metadata_.add_episodes(); + InitCachedEpisode(state_, eh, ce, db_); + } + } + + SegmentEvents(events); + Save(events, updates); +} + +void DayTable::Day::UpdateEpisodes( + const vector& episode_ids, vector* events, const DBHandle& updates) { + for (int i = 0; i < episode_ids.size(); i++) { + // Look for existing episode with this id to replace that entry. + // NOTE: this is O(N*M), but we expect N & M to be small. + int existing_index = -1; + for (int j = 0; j < metadata_.episodes_size(); ++j) { + if (metadata_.episodes(j).id().local_id() == episode_ids[i]) { + existing_index = j; + break; + } + } + + EpisodeHandle eh = state_->episode_table()->LoadEpisode(episode_ids[i], db_); + if (IsEpisodeFullyLoaded(eh)) { + CachedEpisode* ce; + if (existing_index == -1) { + ce = metadata_.add_episodes(); + } else { + ce = metadata_.mutable_episodes(existing_index); + ce->Clear(); + } + InitCachedEpisode(state_, eh, ce, db_); + } else if (existing_index != -1) { + // In this instance, we found an existing cached episode, but have + // since discovered it shouldn't be in the library, so it must be + // removed; swap with last element and remove last. + ProtoRepeatedFieldRemoveElement(metadata_.mutable_episodes(), existing_index); + } + } + + SegmentEvents(events); + Save(events, updates); +} + +void DayTable::Day::SegmentEvents(vector* events) { + std::sort(metadata_.mutable_episodes()->pointer_begin(), + metadata_.mutable_episodes()->pointer_end(), + CachedEpisodeLessThan()); + + // Create a vector of the cached episodes for segmentation. + vector episodes; + vector shared_episodes; + for (int i = 0; i < metadata_.episodes_size(); ++i) { + const CachedEpisode* ce = &metadata_.episodes(i); + if (ce->in_library()) { + episodes.push_back(ce); + } else { + shared_episodes.push_back(ce); + } + } + std::reverse(episodes.begin(), episodes.end()); + + events->clear(); + + // While there are still episodes: + // - Create event with least recent episode + // - Match any remaining episodes to first episode based on thresholds + // - For each episode (E) added in previous step... + // - Match remaining episodes to (E) based on extended threshold ratio + while (!episodes.empty()) { + // Create event with least recent episode. + const CachedEpisode* ce = episodes.back(); // episodes was reversed, so least recent is last + events->push_back(Event(state_, db_)); + events->back().AddEpisode(ce); + episodes.pop_back(); + + // Match remaining episodes to event based on first episode. + vector matched; + for (int i = 0; i < episodes.size(); ++i) { + if (events->back().CanAddEpisode(ce, episodes[i], 1.0)) { + matched.push_back(episodes[i]); + events->back().AddEpisode(episodes[i]); + episodes.erase(episodes.begin() + i); + --i; + } + } + + // Now, match all remaining episodes to those matched in previous + // step using extended thresholds. + for (int i = 0; i < matched.size(); ++i) { + for (int j = 0; j < episodes.size(); ++j) { + if (events->back().CanAddEpisode(matched[i], episodes[j], Event::kExtendThresholdRatio)) { + events->back().AddEpisode(episodes[j]); + episodes.erase(episodes.begin() + j); + --j; + } + } + } + } + + // For each shared episode, add to the first event which contains + // any photos from the episode. These are used to create trapdoors + // (links) between conversations and events in the library and vice + // versa. + for (int i = 0; i < shared_episodes.size(); ++i) { + const CachedEpisode* ce = shared_episodes[i]; + for (int j = 0; j < events->size(); ++j) { + if ((*events)[j].ContainsPhotosFromEpisode(ce)) { + if (!ce->in_library()) { + (*events)[j].AddEpisode(ce); + } + if (ce->has_viewpoint_id()) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + ce->viewpoint_id(), db_); + // Don't show viewpoint trapdoor in events if viewpoint is removed + // or this is the default viewpoint. + if (vh.get() && !vh->label_removed() && !vh->is_default()) { + CreateEventTrapdoor(events, ce, vh, j); + } + } + } + } + } +} + +bool DayTable::Day::IsEpisodeFullyLoaded(const EpisodeHandle& eh) { + return (eh.get() && + eh->has_timestamp() && + eh->has_earliest_photo_timestamp() && + eh->has_latest_photo_timestamp()); +} + +void DayTable::Day::InitCachedEpisode( + AppState* state, const EpisodeHandle& eh, CachedEpisode* ce, const DBHandle& db, + const std::unordered_set* photo_id_filter) { + ce->mutable_id()->CopyFrom(eh->id()); + if (eh->has_parent_id()) { + ce->mutable_parent_id()->CopyFrom(eh->parent_id()); + } + if (eh->has_viewpoint_id()) { + ce->mutable_viewpoint_id()->CopyFrom(eh->viewpoint_id()); + } + ce->set_user_id(eh->user_id()); + + DCHECK(eh->has_timestamp()); + ce->set_timestamp(eh->timestamp()); + + if (!eh->GetLocation(ce->mutable_location(), ce->mutable_placemark())) { + ce->clear_location(); + ce->clear_placemark(); + } + + DCHECK(eh->has_earliest_photo_timestamp()); + ce->set_earliest_photo_timestamp(eh->earliest_photo_timestamp()); + + DCHECK(eh->has_latest_photo_timestamp()); + ce->set_latest_photo_timestamp(eh->latest_photo_timestamp()); + + vector photo_ids; + eh->ListAllPhotos(&photo_ids); + for (int i = 0; i < photo_ids.size(); ++i) { + // Skip photo states which we should never display. + if (eh->IsQuarantined(photo_ids[i]) || + eh->IsRemoved(photo_ids[i]) || + eh->IsUnshared(photo_ids[i])) { + continue; + } + PhotoHandle ph = state->photo_table()->LoadPhoto(photo_ids[i], db); + if (!ph.get() || + (photo_id_filter && !ContainsKey(*photo_id_filter, ph->id().server_id()))) { + continue; + } + DayPhoto* photo = ce->add_photos(); + photo->set_photo_id(ph->id().local_id()); + photo->set_episode_id(eh->id().local_id()); + photo->set_aspect_ratio(ph->aspect_ratio()); + photo->set_timestamp(ph->timestamp()); + } + + ce->set_in_library(eh->InLibrary()); +} + +void DayTable::Day::CreateEventTrapdoor( + vector* events, const CachedEpisode* ce, + const ViewpointHandle& vh, int ev_index) { + const int64_t vp_id = vh->id().local_id(); + TrapdoorHandle trap; + bool found = false; + for (int i = 0; i < (*events)[ev_index].trapdoors_.size(); ++i) { + if ((*events)[ev_index].trapdoors_[i]->viewpoint_id() == vp_id) { + trap = (*events)[ev_index].trapdoors_[i]; + found = true; + break; + } + } + if (!found) { + trap = TrapdoorHandle(new Trapdoor(state_, db_)); + trap->set_viewpoint_id(vp_id); + trap->set_type(Trapdoor::EVENT); + trap->set_event_index(ev_index); + trap->viewpoint_ = vh; + (*events)[ev_index].trapdoors_.push_back(trap); + } + + // Lookup the activity which shared this episode. + vector activity_ids; + state_->activity_table()->ListEpisodeActivities(ce->id().server_id(), &activity_ids, db_); + if (activity_ids.size() > 0) { + ActivityHandle ah = state_->activity_table()->LoadActivity(activity_ids[0], db_); + if (ah.get()) { + DCHECK(ah->has_share_new() || ah->has_share_existing()); + trap->AddSharedEpisode(ah, ce); + } + } +} + + +//// +// Summary + +const float DayTable::Summary::kPhotoVolumeWeightFactor = 2; +const float DayTable::Summary::kCommentVolumeWeightFactor = 1.5; +const float DayTable::Summary::kContributorWeightFactor = 1; +const float DayTable::Summary::kShareWeightFactor = 2; +const float DayTable::Summary::kDistanceWeightFactor = 2; +// TODO(spencer): This value is set so that any unviewed row will have a weight +// that is greater than any viewed row (sum-of-the-weight-factors * +// holiday-multiplier). Take another look at these values. +const float DayTable::Summary::kUnviewedWeightBonus = 30; + +typedef google::protobuf::RepeatedPtrField SummaryRowArray; + +DayTable::Summary::Summary(DayTable* day_table) + : day_table_(day_table), + photo_count_max_(0), + comment_count_max_(0), + contributor_count_max_(0), + share_count_max_(0), + distance_max_(0) { +} + +DayTable::Summary::~Summary() { +} + +bool DayTable::Summary::GetSummaryRow(int row_index, SummaryRow* row) const { + if (row_index < 0 || row_index >= row_count()) { + LOG("requested row index out of bounds %d from %d-%d", + row_index, 0, row_count()); + return false; + } + + row->CopyFrom(summary_.rows(row_index)); + return true; +} + +int DayTable::Summary::GetSummaryRowIndex(WallTime timestamp, int64_t identifier) const { + DCHECK_EQ(timestamp, int(timestamp)); + SummaryRow search_row; + search_row.set_day_timestamp(timestamp); + search_row.set_identifier(identifier); + SummaryRowArray::const_iterator it = + std::lower_bound(summary_.rows().begin(), + summary_.rows().end(), + search_row, SummaryRowGreaterThan()); + if (it != summary_.rows().end() && + it->day_timestamp() == timestamp && + it->identifier() == identifier) { + return it - summary_.rows().begin(); + } + return -1; +} + +void DayTable::Summary::AddDayRows( + WallTime timestamp, const vector& rows) { + DCHECK_EQ(timestamp, int(timestamp)); + if (rows.empty()) { + return; + } + for (int i = 0; i < rows.size(); ++i) { + summary_.mutable_rows()->Add()->CopyFrom(rows[i]); + } + // Sort newly-added values. + std::sort(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + SummaryRowGreaterThan()); +} + +void DayTable::Summary::RemoveDayRows(WallTime timestamp) { + DCHECK_EQ(timestamp, int(timestamp)); + SummaryRow start_row; + start_row.set_day_timestamp(timestamp); + start_row.set_identifier(0); + SummaryRow end_row; + end_row.set_day_timestamp(timestamp - kDayInSeconds); + end_row.set_identifier(0); + SummaryRowArray::pointer_iterator start_it = + std::lower_bound(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + &start_row, SummaryRowGreaterThan()); + SummaryRowArray::pointer_iterator end_it = + std::lower_bound(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + &end_row, SummaryRowGreaterThan()); + const int count = end_it - start_it; + for (SummaryRowArray::pointer_iterator iter = start_it; iter != end_it; ++iter) { + // Set timestamp to -1 so removed rows sort to end. + (*iter)->set_day_timestamp(-1); + } + // Sort removed values to end and truncate. + std::sort(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + SummaryRowGreaterThan()); + for (int i = 0; i < count; ++i) { + DCHECK_EQ(summary_.rows(summary_.rows_size() - 1).day_timestamp(), -1); + if (summary_.rows(summary_.rows_size() - 1).day_timestamp() == -1) { + summary_.mutable_rows()->RemoveLast(); + } + } +} + +void DayTable::Summary::AddRow(const SummaryRow& row) { + summary_.mutable_rows()->Add()->CopyFrom(row); + // Sort newly-added row. + std::sort(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + SummaryRowGreaterThan()); +} + +void DayTable::Summary::RemoveRow(int index) { + CHECK(index >= 0 && index < summary_.rows_size()) + << "day table: trying to remove non-existent summary row " << index; + const int start_size = summary_.rows_size(); + // Swap row to end and remove last. + ProtoRepeatedFieldRemoveElement(summary_.mutable_rows(), index); + // Sort swapped row back into order. + std::sort(summary_.mutable_rows()->pointer_begin(), + summary_.mutable_rows()->pointer_end(), + SummaryRowGreaterThan()); + DCHECK_EQ(summary_.rows_size(), start_size - 1) + << "after removing index " << index << " size went from " + << start_size << " to " << summary_.rows_size(); +} + +bool DayTable::Summary::Load(const string& key, const DBHandle& db) { + db_ = db; + if (!db_->GetProto(key, &summary_)) { + VLOG("day table: failed to load summary info %s", DBIntrospect::Format(key)); + return false; + } + return true; +} + +void DayTable::Summary::Save( + const string& key, const DBHandle& updates) { + Normalize(); + updates->PutProto(key, summary_); +} + +void DayTable::Summary::Normalize() { + summary_.set_total_height(height_prefix()); + summary_.set_photo_count(0); + summary_.set_unviewed_count(0); + + photo_count_max_ = 0; + comment_count_max_ = 0; + contributor_count_max_ = 0; + share_count_max_ = 0; + distance_max_ = 0; + + // Get count maximums and compute row positions & total height. + for (int i = 0; i < summary_.rows_size(); ++i) { + SummaryRow* row = summary_.mutable_rows(i); + row->set_position(summary_.total_height()); + summary_.set_photo_count(summary_.photo_count() + row->photo_count()); + summary_.set_total_height(summary_.total_height() + row->height()); + + photo_count_max_ = + std::max(photo_count_max_, row->photo_count()); + comment_count_max_ = + std::max(comment_count_max_, row->comment_count()); + contributor_count_max_ = + std::max(contributor_count_max_, row->contributor_count()); + share_count_max_ = + std::max(share_count_max_, row->share_count()); + distance_max_ = + std::max(distance_max_, row->distance()); + if (row->unviewed()) { + summary_.set_unviewed_count(summary_.unviewed_count() + 1); + } + } + summary_.set_total_height(summary_.total_height() + height_suffix()); + + // Normalize row weights. + WallTime last_day_timestamp = 0; + bool is_holiday = false; + for (int i = 0; i < summary_.rows_size(); ++i) { + SummaryRow* row = summary_.mutable_rows(i); + // Get the set of days which are holidays. + if (row->day_timestamp() != last_day_timestamp) { + is_holiday = state()->day_table()->IsHoliday(row->day_timestamp(), NULL); + last_day_timestamp = row->day_timestamp(); + } + + NormalizeRowWeight(row, is_holiday); + } +} + +float DayTable::Summary::ComputeWeight(float value, float max, bool log_scale) const { + if (value <= (log_scale ? 1 : 0) || max <= (log_scale ? 1 : 0)) { + return 0; + } + if (log_scale) { + return log(value) / log(max); + } + return value / max; +} + +void DayTable::Summary::NormalizeRowWeight(SummaryRow* row, bool is_holiday) const { + const float photo_weight = + ComputeWeight(row->photo_count(), photo_count_max_, true); + const float comment_weight = + ComputeWeight(row->comment_count(), comment_count_max_, true); + const float contributor_weight = + ComputeWeight(row->contributor_count(), contributor_count_max_, true); + const float share_weight = + ComputeWeight(row->share_count(), share_count_max_, true); + const float distance_weight = + ComputeWeight(row->distance(), distance_max_, true); + + float weight = photo_weight * kPhotoVolumeWeightFactor + + comment_weight * kCommentVolumeWeightFactor + + contributor_weight * kContributorWeightFactor + + share_weight * kShareWeightFactor + + distance_weight * kDistanceWeightFactor; + // Unviewed weight bonus. + if (row->unviewed()) { + weight += kUnviewedWeightBonus; + } + // 1.5x weight multiplier if a holiday. + if (is_holiday) { + weight *= 1.5; + } + row->set_weight(weight); +} + + +//// +// EventSummary + +DayTable::EventSummary::EventSummary(DayTable* day_table) + : Summary(day_table) { +} + +int DayTable::EventSummary::GetEpisodeRowIndex(int64_t episode_id) const { + string key; + if (db_->Get(EncodeEpisodeEventKey(episode_id), &key)) { + WallTime timestamp; + int64_t index; + if (DecodeTimestampAndIdentifier(key, ×tamp, &index)) { + return GetSummaryRowIndex(timestamp, index); + } else { + LOG("day table: failed to decode key %s", key); + } + } + LOG("day table: failed to find row index for episode %d", episode_id); + return -1; +} + +void DayTable::EventSummary::GetViewpointRowIndexes(int64_t viewpoint_id, vector* row_indexes) const { + for (DB::PrefixIterator iter(db_, DBFormat::trapdoor_event_key(viewpoint_id, "")); + iter.Valid(); + iter.Next()) { + WallTime timestamp; + int64_t index; + if (DecodeTimestampAndIdentifier(iter.value(), ×tamp, &index)) { + row_indexes->push_back(GetSummaryRowIndex(timestamp, index)); + } else { + LOG("day table: failed to decode key %s", iter.value()); + } + } +} + +void DayTable::EventSummary::UpdateDay( + WallTime timestamp, const vector& events, + const DBHandle& updates) { + RemoveDayRows(timestamp); + + vector rows(events.size()); + for (int i = 0; i < events.size(); ++i) { + const Event& ev = events[i]; + + SummaryRow& row = rows[i]; + row.set_type(SummaryRow::EVENT); + row.set_timestamp(ev.earliest_timestamp()); + row.set_day_timestamp(int(timestamp)); + row.set_identifier(i); + row.set_height(env()->GetSummaryEventHeight(ev, db_)); + row.set_photo_count(ev.photo_count()); + row.set_contributor_count(ev.contributors_size()); + row.set_distance(ev.distance()); + if (ev.episodes_size() > 0) { + row.set_episode_id(ev.episodes(0).episode_id()); + } + + const string key = EncodeTimestampAndIdentifier(timestamp, i); + int share_count = 0; + for (int j = 0; j < ev.trapdoors().size(); ++j) { + share_count += ev.trapdoors()[j]->photo_count() * ev.trapdoors()[j]->contributors_size(); + updates->Put(EncodeTrapdoorEventKey(ev.trapdoors()[j]->viewpoint_id(), key), key); + } + row.set_share_count(share_count); + + // Add secondary index for all episode ids in the event pointing to the event's key. + for (int j = 0; j < ev.episodes_size(); ++j) { + updates->Put(EncodeEpisodeEventKey(ev.episodes(j).episode_id()), key); + } + } + + AddDayRows(timestamp, rows); +} + +bool DayTable::EventSummary::Load(const DBHandle& db) { + return Summary::Load(kEventSummaryKey, db); +} + +void DayTable::EventSummary::Save(const DBHandle& updates) { + Summary::Save(kEventSummaryKey, updates); +} + + +//// +// FullEventSummary + +DayTable::FullEventSummary::FullEventSummary(DayTable* day_table) + : Summary(day_table) { +} + +int DayTable::FullEventSummary::GetEpisodeRowIndex(int64_t episode_id) const { + string key; + if (db_->Get(EncodeEpisodeEventKey(episode_id), &key)) { + WallTime timestamp; + int64_t index; + if (DecodeTimestampAndIdentifier(key, ×tamp, &index)) { + return GetSummaryRowIndex(timestamp, index); + } else { + LOG("day table: failed to decode key %s", key); + } + } + LOG("day table: failed to find row index for episode %d", episode_id); + return -1; +} + +void DayTable::FullEventSummary::GetViewpointRowIndexes(int64_t viewpoint_id, vector* row_indexes) const { + for (DB::PrefixIterator iter(db_, DBFormat::trapdoor_event_key(viewpoint_id, "")); + iter.Valid(); + iter.Next()) { + WallTime timestamp; + int64_t index; + if (DecodeTimestampAndIdentifier(iter.value(), ×tamp, &index)) { + row_indexes->push_back(GetSummaryRowIndex(timestamp, index)); + } else { + LOG("day table: failed to decode key %s", iter.value()); + } + } +} + +void DayTable::FullEventSummary::UpdateDay( + WallTime timestamp, const vector& events, + const DBHandle& updates) { + RemoveDayRows(timestamp); + + vector rows(events.size()); + for (int i = 0; i < events.size(); ++i) { + const Event& ev = events[i]; + + SummaryRow& row = rows[i]; + row.set_type(SummaryRow::FULL_EVENT); + row.set_timestamp(ev.earliest_timestamp()); + row.set_day_timestamp(int(timestamp)); + row.set_identifier(i); + row.set_height(env()->GetFullEventHeight(ev, db_)); + row.set_photo_count(ev.photo_count()); + row.set_contributor_count(ev.contributors_size()); + row.set_distance(ev.distance()); + if (ev.episodes_size() > 0) { + row.set_episode_id(ev.episodes(0).episode_id()); + } + + int share_count = 0; + for (int j = 0; j < ev.trapdoors().size(); ++j) { + share_count += ev.trapdoors()[j]->photo_count() * ev.trapdoors()[j]->contributors_size(); + } + row.set_share_count(share_count); + + // Secondary index for all episode ids is handled by the normal EventSummary. + } + + AddDayRows(timestamp, rows); +} + +float DayTable::FullEventSummary::height_suffix() const { + return env()->full_event_summary_height_suffix(); +} + +bool DayTable::FullEventSummary::Load(const DBHandle& db) { + return Summary::Load(kFullEventSummaryKey, db); +} + +void DayTable::FullEventSummary::Save(const DBHandle& updates) { + Summary::Save(kFullEventSummaryKey, updates); +} + + +//// +// ConversationSummary + +DayTable::ConversationSummary::ConversationSummary(DayTable* day_table) + : Summary(day_table) { +} + +int DayTable::ConversationSummary::GetViewpointRowIndex(int64_t viewpoint_id) const { + string key; + if (db_->Get(EncodeViewpointConversationKey(viewpoint_id), &key)) { + WallTime timestamp; + int64_t identifier; + if (DecodeTimestampAndIdentifier(key, ×tamp, &identifier)) { + const int row_index = GetSummaryRowIndex(timestamp, identifier); + if (row_index == -1) { + // TODO(spencer): while there is a bug which sometimes causes the + // summary protobuf to get out of date, do a linear search for the + // missing viewpoint. + for (int i = 0; i < summary_.rows_size(); ++i) { + if (summary_.rows(i).identifier() == identifier) { + LOG("found requested viewpoint with timestamp mismatch (%d != %d): %s", + timestamp, summary_.rows(i).day_timestamp(), summary_.rows(i)); + return i; + } + } + } + return row_index; + } else { + LOG("day table: failed to decode key %s", key); + } + } + LOG("day table: failed to find row index for viewpoint %d", viewpoint_id); + return -1; +} + +void DayTable::ConversationSummary::UpdateTrapdoor( + const Trapdoor& trap, const DBHandle& updates) { + if (trap.type() != TrapdoorMetadata::INBOX) { + return; + } + RemoveTrapdoor(trap.viewpoint_id(), updates); + + SummaryRow row; + row.set_type(SummaryRow::TRAPDOOR); + row.set_timestamp(trap.latest_timestamp()); + row.set_day_timestamp(int(trap.latest_timestamp())); + row.set_identifier(trap.viewpoint_id()); + row.set_height(env()->GetInboxCardHeight(trap)); + row.set_photo_count(trap.photo_count()); + row.set_comment_count(trap.comment_count()); + row.set_contributor_count(trap.contributors_size()); + if (trap.unviewed_content()) { + row.set_unviewed(true); + } + + AddRow(row); + const string key = EncodeTimestampAndIdentifier(trap.latest_timestamp(), trap.viewpoint_id()); + updates->Put(EncodeViewpointConversationKey(trap.viewpoint_id()), key); +} + +void DayTable::ConversationSummary::RemoveTrapdoor( + int64_t viewpoint_id, const DBHandle& updates) { + // Do a linear search for the viewpoint in the rows vector. + bool found = true; + for (int i = 0; i < summary_.rows_size(); ++i) { + const SummaryRow& row = summary_.rows(i); + if (row.identifier() == viewpoint_id) { + RemoveRow(i); + SanityCheckRemoved(viewpoint_id); + found = true; + break; + } + } + if (!found) { + LOG("unable to find viewpoint %d in conversation summary:\n%s", + viewpoint_id, summary_); + } + updates->Delete(EncodeViewpointConversationKey(viewpoint_id)); +} + +bool DayTable::ConversationSummary::Load(const DBHandle& db) { + const bool res = Summary::Load(kConversationSummaryKey, db); + if (!res) { + return res; + } + SanityCheck(db); + return res; +} + +void DayTable::ConversationSummary::Save(const DBHandle& updates) { + SanityCheck(updates); + Summary::Save(kConversationSummaryKey, updates); +} + +void DayTable::ConversationSummary::SanityCheck(const DBHandle& db) { +#if defined(ADHOC) || defined(DEVELOPMENT) + // Verify no duplicate viewpoints. + std::unordered_set viewpoint_ids; + for (int i = 0; i < summary_.rows_size(); ++i) { + const SummaryRow& r = summary_.rows(i); + if (ContainsKey(viewpoint_ids, r.identifier())) { + DIE("conversation summary contains duplicate viewpoint %d\n%s", + r.identifier(), summary_); + } + viewpoint_ids.insert(r.identifier()); + } + // Verify viewpoint conversation keys against summary rows. + for (DB::PrefixIterator iter(db, DBFormat::viewpoint_conversation_key("")); + iter.Valid(); + iter.Next()) { + WallTime timestamp; + int64_t identifier; + if (!DecodeTimestampAndIdentifier(iter.value(), ×tamp, &identifier)) { + DIE("invalid timestamp and identifier key for viewpoint conversation %s: %s", + iter.key(), iter.value()); + } + int index = GetSummaryRowIndex(timestamp, identifier); + if (index == -1) { + DIE("unable to locate summary row index for %d/%d in summary: %s", + int(timestamp), identifier, summary_); + } + } +#endif // defined(ADHOC) || defined(DEVELOPMENT) +} + +void DayTable::ConversationSummary::SanityCheckRemoved(int64_t viewpoint_id) { +#if defined(ADHOC) || defined(DEVELOPMENT) + std::unordered_set viewpoint_ids; + for (int i = 0; i < summary_.rows_size(); ++i) { + const SummaryRow& r = summary_.rows(i); + if (viewpoint_id == r.identifier()) { + DIE("conversation summary unexpectedly contains viewpoint %d\n%s", + viewpoint_id, summary_); + } + } +#endif // defined(ADHOC) || defined(DEVELOPMENT) +} + + +//// +// UnviewedConversationSummary + +DayTable::UnviewedConversationSummary::UnviewedConversationSummary(DayTable* day_table) + : Summary(day_table) { +} + +void DayTable::UnviewedConversationSummary::UpdateTrapdoor( + const Trapdoor& trap, const DBHandle& updates) { + RemoveTrapdoor(trap.viewpoint_id(), updates); + + if (trap.type() != TrapdoorMetadata::INBOX || !trap.unviewed_content()) { + return; + } + + SummaryRow row; + row.set_type(SummaryRow::TRAPDOOR); + row.set_timestamp(trap.latest_timestamp()); + row.set_day_timestamp(int(trap.latest_timestamp())); + row.set_identifier(trap.viewpoint_id()); + row.set_height(env()->GetInboxCardHeight(trap)); + row.set_photo_count(trap.photo_count()); + row.set_comment_count(trap.comment_count()); + row.set_contributor_count(trap.contributors_size()); + row.set_unviewed(true); + + AddRow(row); +} + +void DayTable::UnviewedConversationSummary::RemoveTrapdoor( + int64_t viewpoint_id, const DBHandle& updates) { + // Do a linear search for the viewpoint in the rows vector. + for (int i = 0; i < summary_.rows_size(); ++i) { + const SummaryRow& row = summary_.rows(i); + if (row.identifier() == viewpoint_id) { + RemoveRow(i); + break; + } + } +} + +bool DayTable::UnviewedConversationSummary::Load(const DBHandle& db) { + return Summary::Load(kUnviewedConversationSummaryKey, db); +} + +void DayTable::UnviewedConversationSummary::Save(const DBHandle& updates) { + Summary::Save(kUnviewedConversationSummaryKey, updates); +} + + +//// +// ViewpointSummary + +DayTable::ViewpointSummary::ViewpointSummary(DayTable* day_table, const DBHandle& db) + : day_table_(day_table), + db_(db), + total_height_(0) { +} + +DayTable::ViewpointSummary::~ViewpointSummary() { +} + +bool DayTable::ViewpointSummary::Load(int64_t viewpoint_id) { + const string key = EncodeViewpointSummaryKey(viewpoint_id); + if (!db_->Exists(key) || !db_->GetProto(key, this)) { + return false; + } + + return true; +} + +void DayTable::ViewpointSummary::Save( + const DBHandle& updates, Trapdoor* trap) { + // The viewpoint summary is potentially large, as it contains info + // on each activity row in the conversation. + const string summary_key = EncodeViewpointSummaryKey(viewpoint_id()); + updates->PutProto(summary_key, *this); + + // The viewpoint trapdoor metadata is a constant size and is meant + // to be efficiently loaded from the inbox view. + trap->InitFromViewpointSummary(*this); + const string trapdoor_key = EncodeTrapdoorKey(viewpoint_id()); + updates->PutProto(trapdoor_key, *trap); +} + +void DayTable::ViewpointSummary::Rebuild(const ViewpointHandle& vh) { + Clear(); + + if (vh->is_default()) { + return; + } + set_viewpoint_id(vh->id().local_id()); + + ScopedPtr iter( + state()->activity_table()->NewViewpointActivityIterator( + vh->id().local_id(), 0, false, db_)); + PhotoIdSet unique_ids; + ActivityHandle ah; + ActivityHandle prev_ah; + ActivityHandle next_ah; + // Get first visible activity. + for (; !iter->done(); iter->Next()) { + ah = iter->GetActivity(); + if (!ah.get() || ah->IsVisible()) { + break; + } + ah.reset(); // clear for the case of !IsVisible() + } + // Initialize header rows. + if (ah.get()) { + AppendHeaderRow(vh, ah); + } + // Process all activities sequentially. + while (ah.get() != NULL) { + do { + iter->Next(); + next_ah = iter->done() ? ActivityHandle() : iter->GetActivity(); + } while (next_ah.get() && !next_ah->IsVisible()); + AppendActivityRows(vh, ah, prev_ah, next_ah, &unique_ids); + prev_ah = ah; + ah = next_ah; + } + + Normalize(vh); +} + +void DayTable::ViewpointSummary::UpdateActivities( + const ViewpointHandle& vh, const vector& ah_vec) { + if (vh->is_default()) { + return; + } + + // We rebuild the activity array if replacing an existing activity + // or inserting into (as opposed to appending to) the list of activities. + ActivityRowArray existing; + existing.Swap(mutable_activities()); + PhotoIdSet unique_ids; + + ActivityMergeIterator iter(state(), db_, &existing, ah_vec); + if (!iter.done()) { + AppendHeaderRow(vh, iter.cur_ah()); + } + for (; !iter.done(); iter.Next()) { + // Append activity row. + if (iter.should_append()) { + AppendActivityRows(vh, iter.cur_ah(), iter.prev_ah(), iter.next_ah(), &unique_ids); + } else { + // Copy from existing. + add_activities()->CopyFrom(iter.cur_activity()); + // Augment unique ids. + for (int j = 0; j < iter.cur_activity().photos_size(); ++j) { + unique_ids.insert(iter.cur_activity().photos(j).photo_id()); + } + } + } + + Normalize(vh); +} + +void DayTable::ViewpointSummary::UpdateRowHeights(const ViewpointHandle& vh) { + if (!dispatch_is_main_thread()) { + return; + } + for (int i = 0; i < activities_size(); ++i) { + ActivityRow* row = mutable_activities(i); + if (row->type() == HEADER && row->height() == 0) { + row->set_height(env()->GetConversationHeaderHeight(vh, cover_photo().photo_id())); + break; + } + } +} + +void DayTable::ViewpointSummary::UpdateRowPositions() { + total_height_ = 0; + for (int i = 0; i < activities_size(); ++i) { + ActivityRow* row = mutable_activities(i); + row->set_position(total_height_); + total_height_ += row->height(); + } +} + +void DayTable::ViewpointSummary::Delete(int64_t id, const DBHandle& updates) { + const string summary_key = EncodeViewpointSummaryKey(id); + updates->Delete(summary_key); + const string trapdoor_key = EncodeTrapdoorKey(id); + updates->Delete(trapdoor_key); +} + +bool DayTable::ViewpointSummary::IsEmpty() const { + return !provisional() && !photo_count() && !comment_count() && !contributors_size(); +} + +void DayTable::ViewpointSummary::Normalize( + const ViewpointHandle& vh) { + typedef std::unordered_map ContributorMap; + ContributorMap contrib_map; + typedef std::unordered_map ContributorByIdentityMap; + ContributorByIdentityMap contrib_by_identity; + std::unordered_map photo_id_to_episode_id; + int row_count = 0; + WallTime earliest_timestamp = 0; + WallTime latest_timestamp = 0; + int new_comment_count = 0; + int comment_count = 0; + int new_photo_count = 0; + int photo_count = 0; + int prev_row_type = -1; + int non_header_row_count = 0; + + clear_scroll_to_row(); + for (int i = 0; i < activities_size(); ++i) { + ActivityRow* row = mutable_activities(i); + + // Build contributor map. Note that we include even zero height rows + // to ensure that we don't skip users who are part of a conversation + // but whose original add (e.g. in the case of a share_new without + // photos) is a row which isn't displayed. + for (int j = 0; j < row->user_ids_size(); ++j) { + contrib_map[row->user_ids(j)] = row->update_seq(); + } + for (int j = 0; j < row->user_identities_size(); ++j) { + contrib_by_identity[row->user_identities(j)] = row->update_seq(); + } + + if (row->type() != HEADER) { + if (row->height() == 0) { + continue; + } + ++non_header_row_count; + } + if (i == 0 || row->timestamp() < earliest_timestamp) { + earliest_timestamp = row->timestamp(); + } + if (i == 0 || row->timestamp() > latest_timestamp) { + latest_timestamp = row->timestamp(); + } + + // Build map from photo_id -> episode_id. + if (row->type() == PHOTOS || row->type() == HEADER) { + for (int j = 0; j < row->photos_size(); ++j) { + if (row->photos(j).episode_id() != 0) { + photo_id_to_episode_id[row->photos(j).photo_id()] = row->photos(j).episode_id(); + // Check for a duplicate photo also shown as cover photo. In + // this case, subtract one from the photo count. + if (row->type() != HEADER && + cover_photo().photo_id() == row->photos(j).photo_id()) { + --photo_count; + } + } + } + } + // Set any missing episode id for reply-to-photos. + if (row->type() == REPLY_ACTIVITY) { + DCHECK_EQ(row->photos_size(), 1); + if (row->photos(0).episode_id() == 0) { + const int64_t episode_id = FindOrDefault(photo_id_to_episode_id, row->photos(0).photo_id(), 0); + if (episode_id != 0) { + row->mutable_photos(0)->set_episode_id(episode_id); + } else { + // We weren't able to get a corresponding episode id, most + // likely the result of the original photo having been + // unshared, or not yet loaded. Reset the row type and + // height. + ActivityHandle ah = state()->activity_table()->LoadActivity(row->activity_id(), db_); + row->set_height(env()->GetConversationActivityHeight( + vh, ah, -1, + static_cast(row->thread_type()), db_)); + } + } + } + + if (IsThreadTypeCombine(static_cast(row->thread_type())) || + (row->type() == UPDATE && prev_row_type == UPDATE)) { + row_count--; + } + row->set_row_count(row_count++); + prev_row_type = row->type(); + + if (row->update_seq() > vh->viewed_seq()) { + if (!has_scroll_to_row()) { + // We don't keep track of a viewed sequence number on the header + // row, but if the user hasn't seen the first activity, then we + // want to show the entire header (cover photo, title, followers, + // etc.). + set_scroll_to_row(non_header_row_count == 1 ? 0 : i); + } + if (row->type() == PHOTOS || row->type() == HEADER) { + new_photo_count += row->photos_size(); + } else if (row->is_comment()) { + new_comment_count += 1; + } + } + if (row->type() == PHOTOS || row->type() == HEADER) { + photo_count += row->photos_size(); + } else if (row->is_comment()) { + comment_count += 1; + } + } + set_earliest_timestamp(earliest_timestamp); + set_latest_timestamp(latest_timestamp); + set_new_comment_count(new_comment_count); + set_comment_count(comment_count); + set_new_photo_count(new_photo_count); + set_photo_count(photo_count); + set_provisional(vh->provisional()); + + // List followers in order to filter any which were removed. + vector follower_ids; + vh->ListFollowers(&follower_ids); + std::unordered_set follower_set(follower_ids.begin(), follower_ids.end()); + + // Sort list of contributors by most recent contribution. + vector contribs; + for (ContributorMap::iterator iter = contrib_map.begin(); + iter != contrib_map.end(); + ++iter) { + if (ContainsKey(follower_set, iter->first)) { + ViewpointSummaryMetadata::Contributor contrib; + contrib.set_user_id(iter->first); + contrib.set_update_seq(iter->second); + contribs.push_back(contrib); + } + } + for (ContributorByIdentityMap::iterator iter = contrib_by_identity.begin(); + iter != contrib_by_identity.end(); + ++iter) { + ViewpointSummaryMetadata::Contributor contrib; + contrib.set_identity(iter->first); + contrib.set_update_seq(iter->second); + contribs.push_back(contrib); + } + std::sort(contribs.begin(), contribs.end(), ContributorNewerThan()); + + clear_contributors(); + for (int i = 0; i < contribs.size(); ++i) { + add_contributors()->CopyFrom(contribs[i]); + } +} + +void DayTable::ViewpointSummary::AppendHeaderRow( + const ViewpointHandle& vh, const ActivityHandle& ah) { + CHECK_EQ(activities_size(), 0); + + // Try to get cover photo. + clear_cover_photo(); + int64_t cover_photo_id; + int64_t cover_episode_id; + WallTime cover_timestamp; + float cover_aspect_ratio; + + ActivityRow* row = add_activities(); + row->set_activity_id(ah->activity_id().local_id()); + row->set_timestamp(ah->timestamp()); + row->set_type(HEADER); + // The actual height will be computed when the conversation is loaded. + row->set_height(0); + + if (vh->GetCoverPhoto( + &cover_photo_id, &cover_episode_id, &cover_timestamp, &cover_aspect_ratio)) { + ActivityRow::Photo* arp = row->add_photos(); + arp->set_photo_id(cover_photo_id); + arp->set_episode_id(cover_episode_id); + + mutable_cover_photo()->set_photo_id(cover_photo_id); + mutable_cover_photo()->set_episode_id(cover_episode_id); + mutable_cover_photo()->set_timestamp(cover_timestamp); + mutable_cover_photo()->set_aspect_ratio(cover_aspect_ratio); + } +} + +void DayTable::ViewpointSummary::AppendActivityRows( + const ViewpointHandle& vh, const ActivityHandle& ah, + const ActivityHandle& prev_ah, const ActivityHandle& next_ah, + PhotoIdSet* unique_ids) { + DCHECK(ah->IsVisible()); + // Check if this is the first non-header row. + CHECK_GT(activities_size(), 0) << "the header row should have been appended"; + bool first_activity = true; + for (int i = 1 /* skip header row */; i < activities_size(); ++i) { + if (activities(i).height() > 0) { + first_activity = false; + } + } + bool empty = true; + + const int start_activity = activities_size(); + ActivityRow* row = add_activities(); + row->set_activity_id(ah->activity_id().local_id()); + row->set_timestamp(ah->timestamp()); + row->set_type(ACTIVITY); + row->set_thread_type(THREAD_NONE); + if (ah->provisional()) { + row->set_is_provisional_hint(true); + } + + if (ah->has_share_new() || ah->has_share_existing()) { + row->set_thread_type(THREAD_PHOTOS); + row->set_height(env()->GetConversationActivityHeight(vh, ah, -1, THREAD_PHOTOS, db_)); + + // Build list of photos to display from this activity. + int duplicate_cover_photo_index = -1; + vector photos; + vector episodes; + const ShareEpisodes* share_episodes = ah->GetShareEpisodes(); + for (int i = 0; i < share_episodes->size(); ++i) { + const EpisodeId& episode_id = share_episodes->Get(i).episode_id(); + EpisodeHandle eh = state()->episode_table()->LoadEpisode(episode_id, db_); + if (!eh.get()) { + LOG("day table: couldn't get episode %d", episode_id); + continue; + } + + // Exclude unshared ids. + vector unshared_ids; + eh->ListUnshared(&unshared_ids); + std::unordered_set unshared_set(unshared_ids.begin(), unshared_ids.end()); + + for (int j = 0; j < share_episodes->Get(i).photo_ids_size(); ++j) { + PhotoHandle ph = state()->photo_table()->LoadPhoto( + share_episodes->Get(i).photo_ids(j), db_); + const int64_t photo_id = (ph.get() && !ph->label_error()) ? ph->id().local_id() : -1; + // Exclude photos which can't be loaded, any photos which + // have been unshared, and photos which have already been displayed. + if (photo_id != -1 && + !ContainsKey(unshared_set, photo_id) && + !ContainsKey(*unique_ids, photo_id)) { + unique_ids->insert(photo_id); + + if (photo_id == cover_photo().photo_id()) { + duplicate_cover_photo_index = photos.size(); + } + + // Add all photos to activity row. + ActivityRow::Photo* arp = row->add_photos(); + arp->set_photo_id(ph->id().local_id()); + arp->set_episode_id(eh->id().local_id()); + if (eh->has_parent_id()) { + arp->set_parent_episode_id(eh->parent_id().local_id()); + } + + photos.push_back(ph); + episodes.push_back(eh); + } + } + } + + // If this is the first visible row and contains only the cover + // photo, don't show the cover photo twice. Otherwise, we allow + // the cover photo to be duplicated in the conversation to ease + // user confusion. + if (duplicate_cover_photo_index != -1 && + first_activity && + photos.size() == 1) { + photos.clear(); + } + + // Skip if there are no photos after unshares are filtered. + if (!photos.empty()) { + DCHECK_EQ(photos.size(), episodes.size()); + empty = false; + + ViewpointSummaryMetadata::ActivityRow* episode_row = add_activities(); + episode_row->set_activity_id(ah->activity_id().local_id()); + episode_row->set_timestamp(ah->timestamp()); + episode_row->set_type(PHOTOS); + + for (int k = 0; k < photos.size(); ++k) { + ActivityRow::Photo* arp = episode_row->add_photos(); + const PhotoHandle& ph = photos[k]; + const EpisodeHandle& eh = episodes[k]; + arp->set_photo_id(ph->id().local_id()); + arp->set_episode_id(eh->id().local_id()); + if (eh->has_parent_id()) { + arp->set_parent_episode_id(eh->parent_id().local_id()); + } + } + episode_row->set_height( + env()->GetShareActivityPhotosRowHeight( + CONVERSATION_LAYOUT, photos, episodes, db_)); + } + } else if (ah->has_post_comment()) { + CommentHandle ch = state()->comment_table()->LoadComment( + ah->post_comment().comment_id(), db_); + if (ch.get()) { + empty = false; + bool prev_comment = prev_ah.get() && prev_ah->has_post_comment(); + double prev_delta = prev_comment ? ah->timestamp() - prev_ah->timestamp() : 0; + bool prev_cont = prev_comment && (prev_delta < kCommentThreadThreshold); + + bool next_comment = next_ah.get() && next_ah->has_post_comment(); + double next_delta = 0; + bool next_cont = false; + if (next_comment) { + // The next comment is only a continuation if it isn't a reply. + CommentHandle next_ch = state()->comment_table()->LoadComment( + next_ah->post_comment().comment_id(), db_); + if (next_ch.get() && !next_ch->has_asset_id()) { + next_delta = next_ah->timestamp() - ah->timestamp(); + next_cont = next_delta < kCommentThreadThreshold; + } + } + + // Handle reply-to-photo comment. + int64_t reply_to_photo_id = -1; + if (ch->has_asset_id()) { + PhotoHandle ph = state()->photo_table()->LoadPhoto(ch->asset_id(), db_); + if (ph.get()) { + reply_to_photo_id = ph->id().local_id(); + prev_cont = false; + row->set_type(REPLY_ACTIVITY); + ActivityRow::Photo* arp = row->add_photos(); + arp->set_photo_id(reply_to_photo_id); + } + } + + if (prev_cont) { + if (ah->user_id() == prev_ah->user_id()) { + // If same user combine comments into a single + // row. However, if the minute is different, show a new + // time indication. + const bool same_minute = int(ah->timestamp() / 60) == int(prev_ah->timestamp() / 60); + if (!next_cont) { + row->set_thread_type(same_minute ? + THREAD_COMBINE_END : THREAD_COMBINE_END_WITH_TIME); + } else if (ah->user_id() == next_ah->user_id()) { + row->set_thread_type(same_minute ? + THREAD_COMBINE : THREAD_COMBINE_WITH_TIME); + } else { + // Otherwise, combine but to a new user's comment. + row->set_thread_type(same_minute ? + THREAD_COMBINE_NEW_USER : THREAD_COMBINE_NEW_USER_WITH_TIME); + } + } else if (!next_cont) { + row->set_thread_type(THREAD_END); + } else { + row->set_thread_type(THREAD_POINT); + } + } else if (!prev_cont && next_cont) { + row->set_thread_type(THREAD_START); + } + row->set_is_comment(true); + row->set_height(env()->GetConversationActivityHeight( + vh, ah, reply_to_photo_id, + static_cast(row->thread_type()), db_)); + } + } else if (ah->IsUpdate()) { + empty = false; + row->set_type(UPDATE); + bool prev_update = prev_ah.get() && prev_ah->IsUpdate(); + bool next_update = next_ah.get() && next_ah->IsUpdate(); + if (!prev_update) { + row->set_thread_type(next_update ? UPDATE_START : UPDATE_SINGLE); + } else { + row->set_thread_type(next_update ? UPDATE_COMBINE : UPDATE_END); + } + row->set_height(env()->GetConversationUpdateHeight( + vh, ah, static_cast(row->thread_type()), db_)); + } + + // Add contributor. + row->add_user_ids(ah->user_id()); + + // Add sharees if available. + typedef ::google::protobuf::RepeatedPtrField ContactArray; + const ContactArray* contacts = ActivityTable::GetActivityContacts(*ah); + for (int i = 0; contacts && i < contacts->size(); ++i) { + if (contacts->Get(i).has_user_id()) { + row->add_user_ids(contacts->Get(i).user_id()); + } else { + DCHECK(!contacts->Get(i).primary_identity().empty()); + row->add_user_identities(contacts->Get(i).primary_identity()); + } + } + + // If an activity couldn't be rendered, usually because comment or + // episode hasn't been fully downloaded yet, set the row height to + // zero, which will maintain its place in the viewpoint summary, but + // prevent it from being displayed and prematurely marked as viewed. + // We do this before setting update sequence to avoid having this + // incomplete row scrolled-to in case the viewpoint is viewed. + if (empty) { + row->set_height(0); + } + + // Set update sequence number. There is a window where activities are + // viewed and have their timestamps set before the viewpoint viewed_seq + // is updated. We handle this case here by just setting the putative + // update_seq for the row to the value of the viewpoint's viewed_seq. + // Also, if the activity is the user's own, there's a window between + // the viewpoint's update_seq and viewed_seq being incremented. + for (int i = start_activity; i < activities_size(); ++i) { + row = mutable_activities(i); + if (row->height() == 0) { + continue; + } + if ((ah->has_viewed_timestamp() || ah->user_id() == state()->user_id()) && + ah->update_seq() > vh->viewed_seq()) { + row->set_update_seq(vh->viewed_seq()); + } else { + row->set_update_seq(ah->update_seq()); + } + // Set pending boolean if we need to upload to server. + if (ah->upload_activity()) { + row->set_pending(true); + } + } +} + + +//// +// Snapshot + +DayTable::Snapshot::Snapshot(AppState* state, const DBHandle& snapshot_db) + : state_(state), + snapshot_db_(snapshot_db) { + events_.reset(new EventSummary(state_->day_table())); + events_->Load(snapshot_db_); + full_events_.reset(new FullEventSummary(state_->day_table())); + full_events_->Load(snapshot_db_); + conversations_.reset(new ConversationSummary(state_->day_table())); + conversations_->Load(snapshot_db_); + unviewed_conversations_.reset(new UnviewedConversationSummary(state_->day_table())); + unviewed_conversations_->Load(snapshot_db_); +} + +DayTable::Snapshot::~Snapshot() { +} + +DayTable::ViewpointSummaryHandle +DayTable::Snapshot::LoadViewpointSummary(int64_t viewpoint_id) const { + ViewpointSummaryHandle vsh( + new ViewpointSummary(state_->day_table(), snapshot_db_)); +#ifdef ALWAYS_REBUILD_CONVERSATIONS + const bool rebuild = true; +#else // ALWAYS_REBUILD_CONVERSATIONS + const bool rebuild = false; +#endif // ALWAYS_REBUILD_CONVERSATIONS + if (rebuild || !vsh->Load(viewpoint_id)) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + viewpoint_id, snapshot_db_); + vsh->Rebuild(vh); + } + return vsh; +} + +EventHandle DayTable::Snapshot::LoadEvent(WallTime timestamp, int index) { + timestamp = CanonicalizeTimestamp(timestamp); + + EventHandle evh(new Event(state_, snapshot_db_)); + if (!snapshot_db_->GetProto(EncodeDayEventKey(timestamp, index), evh.get())) { + LOG("day table: failed to get event at %d/%d", int(timestamp), index); + return EventHandle(); + } + + // Initialize the trapdoor objects. + for (int i = 0; i < evh->trapdoors_size(); ++i) { + evh->trapdoors_.push_back(new Trapdoor(state_, snapshot_db_)); + evh->trapdoors_[i]->Swap(evh->mutable_trapdoors(i)); + } + evh->clear_trapdoors(); + + return evh; +} + +TrapdoorHandle DayTable::Snapshot::LoadTrapdoor(int64_t viewpoint_id) { + TrapdoorHandle trh(new Trapdoor(state_, snapshot_db_)); + if (!snapshot_db_->GetProto(EncodeTrapdoorKey(viewpoint_id), trh.get())) { + LOG("day table: failed to get trapdoor %d", viewpoint_id); + return TrapdoorHandle(); + } + return trh; +} + +TrapdoorHandle DayTable::Snapshot::LoadPhotoTrapdoor( + int64_t viewpoint_id, int64_t photo_id) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_id, snapshot_db_); + if (!ph.get()) { + LOG("day table: failed to get photo for trapdoor %d", photo_id); + return TrapdoorHandle(); + } + TrapdoorHandle trh(new Trapdoor(state_, snapshot_db_)); + if (!snapshot_db_->GetProto(EncodeTrapdoorKey(viewpoint_id), trh.get())) { + LOG("day table: failed to get trapdoor %d", viewpoint_id); + return TrapdoorHandle(); + } + + // Clear sampled photos and add in photo_id as only sampled photo. + trh->clear_photos(); + DayPhoto* photo = trh->add_photos(); + photo->set_photo_id(ph->id().local_id()); + photo->set_episode_id(0); + photo->set_aspect_ratio(ph->aspect_ratio()); + photo->set_timestamp(ph->timestamp()); + + return trh; +} + + +//// +// DayTable + +DayTable::DayTable(AppState* state, DayTableEnv* env) + : state_(state), + env_(env), + epoch_(1), + initialized_(false), + all_refreshes_paused_(false), + event_refreshes_paused_(false), + refreshing_(false) { + CHECK(env != NULL); + + invalidation_seq_no_ = state_->db()->Get(kDayTableInvalidationSeqNoKey, 1); + timezone_ = state_->db()->Get(kDayTableTimezoneKey, ""); + + // Build map from localized holiday timestamp to holiday array index. + for (int i = 0; i < ARRAYSIZE(kUSHolidays); ++i) { + const WallTime timestamp = kUSHolidays[i].timestamp; + const WallTime local_timestamp = + timestamp - state_->TimeZoneOffset(timestamp) + kPracticalDayOffset; + // LOG("%s: %d", kUSHolidays[i].title, int(local_timestamp)); + holiday_timestamps_[local_timestamp] = i; + } +} + +DayTable::~DayTable() { +} + +bool DayTable::Initialize(bool force_reset) { + MutexLock l(&mu_); + initialized_ = true; + bool delayed_refresh = true; + bool reset = false; + +#ifdef RESET_DAY_TABLE + force_reset = true; +#endif // RESET_DAY_TABLE + + if (ShouldUpdateTimezone() || + force_reset || + kDayTableFormat != state_->db()->Get(kDayTableFormatKey, 0)) { + // Immediately delete all existing day table data and invalidate + // activities and episodes as appropriate to rebuild cached day metadata. + DBHandle updates = state_->NewDBTransaction(); + UpdateTimezone(updates); + ResetLocked(updates); + updates->Commit(false); + delayed_refresh = false; + reset = true; + } + + // Always start paused--refreshing is resumed when maintenance is + // completed and dashboard has confirmed initial scan is complete. + all_refreshes_paused_ = true; + event_refreshes_paused_ = true; + + // Take the initial day snapshot after the day table format has been + // verified. If we took the snapshot earlier the snapshot might contain + // unexpected data which could cause crashes or other harmful behavior. + snapshot_.reset(new DayTable::Snapshot(state_, state_->NewDBSnapshot())); + epoch_++; + + // Schedule delayed garbage collection after maintenance has completed. + state_->maintenance_done()->Add([this](bool reset) { + state_->async()->dispatch_after_background(12, [this] { + GarbageCollect(); + }); + }); + + return reset; +} + +bool DayTable::initialized() const { + MutexLock l(&mu_); + return initialized_; +} + +const DayTable::SnapshotHandle& DayTable::GetSnapshot(int* epoch) { + MutexLock l(&mu_); + DCHECK(initialized_); + if (epoch) { + *epoch = epoch_; + } + return snapshot_; +} + +void DayTable::InvalidateActivity( + const ActivityHandle& ah, const DBHandle& updates) { + MutexLock l(&mu_); + if (!initialized_) { + return; + } + invalidation_seq_no_++; + state_->db()->Put(kDayTableInvalidationSeqNoKey, invalidation_seq_no_); + const WallTime timestamp = CanonicalizeTimestamp(ah->timestamp()); + updates->Put(EncodeViewpointInvalidationKey( + timestamp, ah->viewpoint_id().local_id(), ah->activity_id().local_id()), + invalidation_seq_no_); + if (!all_refreshes_paused_) { + updates->AddCommitTrigger(kDayTableCommitTrigger, [this] { + MaybeRefresh(); + }); + } +} + +void DayTable::InvalidateDay(WallTime timestamp, const DBHandle& updates) { + MutexLock l(&mu_); + if (!initialized_) { + return; + } + InvalidateDayLocked(timestamp, updates); + if (!all_refreshes_paused_) { + updates->AddCommitTrigger(kDayTableCommitTrigger, [this] { + MaybeRefresh(); + }); + } +} + +void DayTable::InvalidateDayLocked(WallTime timestamp, const DBHandle& updates) { + invalidation_seq_no_++; + state_->db()->Put(kDayTableInvalidationSeqNoKey, invalidation_seq_no_); + timestamp = CanonicalizeTimestamp(timestamp); + updates->Put(EncodeDayEpisodeInvalidationKey(timestamp, -1), invalidation_seq_no_); +} + +void DayTable::InvalidateEpisode(const EpisodeHandle& eh, const DBHandle& updates) { + MutexLock l(&mu_); + if (!initialized_) { + return; + } + invalidation_seq_no_++; + state_->db()->Put(kDayTableInvalidationSeqNoKey, invalidation_seq_no_); + updates->Put(EncodeDayEpisodeInvalidationKey( + CanonicalizeTimestamp(eh->timestamp()), eh->id().local_id()), + invalidation_seq_no_); + if (!all_refreshes_paused_ && !event_refreshes_paused_) { + updates->AddCommitTrigger(kDayTableCommitTrigger, [this] { + MaybeRefresh(); + }); + } +} + +void DayTable::InvalidateViewpoint(const ViewpointHandle& vh, const DBHandle& updates) { + MutexLock l(&mu_); + if (!initialized_) { + return; + } + InvalidateViewpointLocked(vh, updates); + if (!all_refreshes_paused_) { + updates->AddCommitTrigger(kDayTableCommitTrigger, [this] { + MaybeRefresh(); + }); + } +} + +void DayTable::InvalidateUser(int64_t user_id, const DBHandle& updates) { + MutexLock l(&mu_); + if (!initialized_) { + return; + } + invalidation_seq_no_++; + state_->db()->Put(kDayTableInvalidationSeqNoKey, invalidation_seq_no_); + updates->Put(EncodeUserInvalidationKey(user_id), invalidation_seq_no_); + + if (!all_refreshes_paused_) { + updates->AddCommitTrigger(kDayTableCommitTrigger, [this] { + MaybeRefresh(); + }); + } +} + +void DayTable::InvalidateViewpointLocked( + const ViewpointHandle& vh, const DBHandle& updates) { + // Set the timestamp for this invalidation to the latest activity for the viewpoint. + ActivityHandle ah = state_->activity_table()->GetLatestActivity( + vh->id().local_id(), updates); + if (ah.get()) { + invalidation_seq_no_++; + state_->db()->Put(kDayTableInvalidationSeqNoKey, invalidation_seq_no_); + const WallTime timestamp = CanonicalizeTimestamp(ah->timestamp()); + updates->Put(EncodeViewpointInvalidationKey(timestamp, vh->id().local_id(), -1), + invalidation_seq_no_); + } +} + +void DayTable::InvalidateSnapshot() { + MutexLock l(&mu_); + InvalidateSnapshotLocked(); +} + +void DayTable::InvalidateSnapshotLocked() { + DCHECK(initialized_); + snapshot_.reset(new DayTable::Snapshot(state_, state_->NewDBSnapshot())); + epoch_++; + state_->async()->dispatch_main([this] { + update_.Run(); + }); +} + +void DayTable::PauseAllRefreshes() { + MutexLock l(&mu_); + DCHECK(!all_refreshes_paused_); + all_refreshes_paused_ = true; + mu_.Wait([this] { + return !refreshing_; + }); +} + +void DayTable::PauseEventRefreshes() { + MutexLock l(&mu_); + DCHECK(!event_refreshes_paused_); + event_refreshes_paused_ = true; + mu_.Wait([this] { + return !refreshing_; + }); +} + +void DayTable::ResumeAllRefreshes(Callback callback) { + { + MutexLock l(&mu_); + all_refreshes_paused_ = false; + } + MaybeRefresh(callback); +} + +void DayTable::ResumeEventRefreshes() { + { + MutexLock l(&mu_); + event_refreshes_paused_ = false; + } + MaybeRefresh(); +} + +bool DayTable::refreshing() const { + MutexLock l(&mu_); + return refreshing_; +} + +bool DayTable::IsHoliday(WallTime timestamp, string* s) { + const int index = FindOrDefault(holiday_timestamps_, timestamp, -1); + if (index == -1) { + return false; + } + if (s) { + *s = kUSHolidays[index].title; + } + return true; +} + +bool DayTable::ShouldUpdateTimezone() const { + setenv("TZ", state_->timezone().c_str(), 1); + tzset(); + return (timezone_ != state_->timezone()); +} + +void DayTable::UpdateTimezone(const DBHandle& updates) { + timezone_ = state_->timezone(); + updates->Put(kDayTableTimezoneKey, timezone_); +} + +void DayTable::ResetLocked(const DBHandle& updates) { + DeleteDayTablesLocked(updates); + InvalidateAllDaysLocked(updates); + InvalidateAllViewpointsLocked(updates); +} + +void DayTable::DeleteDayTablesLocked(const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, kDayKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kDayEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kDayEpisodeInvalidationKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kTrapdoorEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kTrapdoorKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kEpisodeEventKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kViewpointConversationKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, kViewpointInvalidationKeyPrefix); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_summary_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + updates->Delete(kEpisodeSummaryKey); + updates->Delete(kEventSummaryKey); + updates->Delete(kConversationSummaryKey); + updates->Delete(kFullEventSummaryKey); + updates->Delete(kUnviewedConversationSummaryKey); + LOG("day table: clearing all cached day summaries"); + updates->Put(kDayTableFormatKey, kDayTableFormat); +} + +void DayTable::InvalidateAllDaysLocked(const DBHandle& updates) { + // Now, create invalidations for all days with content. + ScopedPtr builder_iter(new DayBuilderIterator(state_, updates)); + int days = 0; + while (!builder_iter->done()) { + ++days; + InvalidateDayLocked(builder_iter->timestamp(), updates); + builder_iter->Next(); + } + LOG("day table: invalidated %d days", days); +} + +void DayTable::InvalidateAllViewpointsLocked(const DBHandle& updates) { + int viewpoints = 0; + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t viewpoint_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + viewpoint_id, updates); + InvalidateViewpointLocked(vh, updates); + ++viewpoints; + } + LOG("day table: invalidated %d viewpoints", viewpoints); +} + +void DayTable::DeleteInvalidationKeysLocked( + const vector >& invalidation_keys, + const DBHandle& updates) { + mu_.AssertHeld(); + // With mutex lock, delete invalidation keys if no new invalidation + // arrived and commit, guaranteeing that we don't lose invalidations. + for (int i = 0; i < invalidation_keys.size(); ++i) { + const string& key = invalidation_keys[i].first; + const int64_t snapshot_isn = invalidation_keys[i].second; + const int64_t current_isn = state_->db()->Get(key, -1); + if (current_isn == snapshot_isn) { + updates->Delete(key); + } + } +} + +void DayTable::MaybeRefresh(Callback callback) { + int64_t start_invalidation_seq_no; + { + MutexLock l(&mu_); + if (refreshing_ || all_refreshes_paused_) { + DCHECK(!callback); + return; + } + LOG("day table: starting refresh cycle"); + refreshing_ = true; + start_invalidation_seq_no = invalidation_seq_no_; + } + + WallTimer timer; + state_->async()->dispatch_low_priority( + [this, callback, start_invalidation_seq_no, timer] { + bool callback_invoked = false; + bool done = false; + while (!done) { + bool day_episodes_completed; + bool viewpoints_completed; + bool users_completed; + int refreshed_day_episodes = RefreshDayEpisodes(&day_episodes_completed); + int refreshed_viewpoints = RefreshViewpoints(&viewpoints_completed); + int refreshed_users = RefreshUsers(&users_completed); + + // If there's a callback, invoke it now. + if (callback && !callback_invoked) { + callback(); + callback_invoked = true; + } + + // Check if work was done. + if (refreshed_day_episodes + refreshed_viewpoints + refreshed_users == 0) { + // In case there was no work to be done, backoff for 1 + // second so successive calls to InvalidateDay() don't + // busily generate refresh log messages. + state_->async()->dispatch_after_background( + 1, [this, start_invalidation_seq_no] { + mu_.Lock(); + refreshing_ = false; + // Make sure we notify any listeners that may be waiting for + // the refreshing to become false. + state_->async()->dispatch_main([this] { + update_.Run(); + }); + const bool maybe_refresh = + invalidation_seq_no_ != start_invalidation_seq_no; + mu_.Unlock(); + if (maybe_refresh) { + MaybeRefresh(); + } + }); + done = true; + } else { + MutexLock l(&mu_); + // Break out of loop if we've processed all invalidated days + // and if no new invalidations have occurred. + if (day_episodes_completed && viewpoints_completed && users_completed && + invalidation_seq_no_ == start_invalidation_seq_no) { + refreshing_ = false; + done = true; + } + InvalidateSnapshotLocked(); + } + } + LOG("day table: completed refresh cycle: %.1f sec", timer.Get()); + +#ifdef CONTINUOUS_DAY_TABLE_REFRESH + { + LOG("day table: continuous day table refresh"); + DBHandle updates = state_->NewDBTransaction(); + InvalidateAllDaysLocked(updates); + InvalidateAllViewpointsLocked(updates); + updates->Commit(false); + } +#endif // CONTINUOUS_DAY_TABLE_REFRESH + }); +} + +int DayTable::RefreshDayEpisodes(bool* completed) { + if (event_refreshes_paused_) { + return 0; + } + + WallTimer timer; + DBHandle snap = state_->NewDBSnapshot(); + DBHandle updates = state_->NewDBTransaction(); + + typedef std::unordered_map > InvalidationMap; + InvalidationMap invalidations; + WallTime last_timestamp = 0; + int refreshes = 0; + vector > invalidation_keys; + + *completed = true; // start by assuming we complete full scan + + for (DB::PrefixIterator iter(snap, kDayEpisodeInvalidationKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + // Stop iteration if we exceed the refresh count limit per iteration. + if (refreshes >= kMinRefreshCount) { + *completed = false; + break; + } + WallTime timestamp; + int64_t episode_id; + if (!DecodeDayEpisodeInvalidationKey(key, ×tamp, &episode_id)) { + LOG("day table: unable to decode invalidation sequence key: %s", key); + continue; + } + + if (timestamp != last_timestamp) { + ++refreshes; + last_timestamp = timestamp; + } + + // Build the invalidations map. + invalidations[timestamp].push_back(episode_id); + + // Add invalidation key for deletion under mutex. + const int64_t snapshot_isn = FromString(value); + invalidation_keys.push_back(std::make_pair(key.ToString(), snapshot_isn)); + } + + if (invalidations.empty()) { + return 0; + } + + ScopedPtr event_summary(new EventSummary(this)); + ScopedPtr episode_summary(new FullEventSummary(this)); + event_summary->Load(snap); + episode_summary->Load(snap); + + for (InvalidationMap::iterator iter = invalidations.begin(); + iter != invalidations.end(); + ++iter) { + const WallTime timestamp = iter->first; + const vector& episode_ids = iter->second; + vector events; + Day day(state_, timestamp, snap); + if (!day.Load()) { + day.Rebuild(&events, updates); + } else { + day.UpdateEpisodes(episode_ids, &events, updates); + } + + event_summary->UpdateDay(timestamp, events, updates); + episode_summary->UpdateDay(timestamp, events, updates); + } + + event_summary->Save(updates); + episode_summary->Save(updates); + + MutexLock l(&mu_); + DeleteInvalidationKeysLocked(invalidation_keys, updates); + CHECK(updates->Commit(false)) << "failed database update"; + + if (refreshes > 0) { + LOG("day table: %d day episode refreshes in %.3fs", refreshes, timer.Get()); + } + return refreshes; +} + +int DayTable::RefreshViewpoints(bool* completed) { + WallTimer timer; + DBHandle snap = state_->NewDBSnapshot(); + DBHandle updates = state_->NewDBTransaction(); + typedef std::unordered_map > InvalidationMap; + InvalidationMap invalidations; + int64_t last_viewpoint_id = 0; + int refreshes = 0; + vector > invalidation_keys; + + *completed = true; // start by assuming we complete full scan + + for (DB::PrefixIterator iter(snap, DBFormat::viewpoint_invalidation_key("")); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + // Stop iteration if we exceed the refresh count limit per iteration. + if (refreshes >= kMinRefreshCount) { + *completed = false; + break; + } + WallTime timestamp; + int64_t viewpoint_id; + int64_t activity_id; + if (!DecodeViewpointInvalidationKey(key, ×tamp, &viewpoint_id, &activity_id)) { + LOG("day table: unable to decode viewpoint invalidation key: %s", key); + continue; + } + + if (viewpoint_id != last_viewpoint_id) { + ++refreshes; + last_viewpoint_id = viewpoint_id; + } + + // Build the invalidations map. + invalidations[viewpoint_id].push_back(activity_id); + + // Save invalidation key for deletion under the mutex. + const int64_t snapshot_isn = FromString(value); + invalidation_keys.push_back(std::make_pair(key.ToString(), snapshot_isn)); + } + + if (invalidations.empty()) { + return 0; + } + + ScopedPtr conversation_summary(new ConversationSummary(this)); + ScopedPtr unviewed_conversation_summary( + new UnviewedConversationSummary(this)); + conversation_summary->Load(snap); + unviewed_conversation_summary->Load(snap); + VLOG("day table: loaded conversation summary with %d rows", + conversation_summary->row_count()); + + // Process updates to each viewpoint. + for (InvalidationMap::iterator iter = invalidations.begin(); + iter != invalidations.end(); + ++iter) { + const int64_t viewpoint_id = iter->first; + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(viewpoint_id, snap); + // Skip the user's default viewpoint and any removed viewpoints. + if (!vh.get() || vh->is_default() || vh->label_removed()) { + ViewpointSummary::Delete(viewpoint_id, updates); + conversation_summary->RemoveTrapdoor(viewpoint_id, updates); + unviewed_conversation_summary->RemoveTrapdoor(viewpoint_id, updates); + VLOG("day table: removed trapdoor for viewpoint %d", viewpoint_id); + continue; + } + + ViewpointSummaryHandle vsh(new ViewpointSummary(state_->day_table(), snap)); + if (!vsh->Load(viewpoint_id)) { + vsh->Rebuild(vh); + } else { + // Augment the viewpoint summary with updated activities. + std::unordered_set activity_ids; + vector ah_vec; + for (int i = 0; i < iter->second.size(); ++i) { + // Ignore duplicate activities via multiple invalidations. + if (ContainsKey(activity_ids, iter->second[i])) { + continue; + } + activity_ids.insert(iter->second[i]); + ActivityHandle ah = state_->activity_table()->LoadActivity(iter->second[i], snap); + if (ah.get()) { + ah_vec.push_back(ah); + } + } + std::sort(ah_vec.begin(), ah_vec.end(), ActivityOlderThan()); + vsh->UpdateActivities(vh, ah_vec); + } + + if (!vsh->IsEmpty()) { + Trapdoor trap(state_, snap); + vsh->Save(updates, &trap); + + conversation_summary->UpdateTrapdoor(trap, updates); + unviewed_conversation_summary->UpdateTrapdoor(trap, updates); + VLOG("day table: updated trapdoor for viewpoint %d", viewpoint_id); + } else { + VLOG("day table: skipping empty viewpoint %s", vh->id()); + ViewpointSummary::Delete(viewpoint_id, updates); + conversation_summary->RemoveTrapdoor(viewpoint_id, updates); + unviewed_conversation_summary->RemoveTrapdoor(viewpoint_id, updates); + } + } + + conversation_summary->Save(updates); + unviewed_conversation_summary->Save(updates); + + VLOG("day table: saved conversation summary with %d rows", + conversation_summary->row_count()); + + MutexLock l(&mu_); + DeleteInvalidationKeysLocked(invalidation_keys, updates); + CHECK(updates->Commit(false)) << "failed database update"; + + // Verify conversation summary after save has completed. + conversation_summary->SanityCheck(state_->db()); + + if (refreshes > 0) { + LOG("day table: %d viewpoints refreshed in %.3fs", refreshes, timer.Get()); + } + return refreshes; +} + +int DayTable::RefreshUsers(bool* completed) { + WallTimer timer; + DBHandle snap = state_->NewDBSnapshot(); + DBHandle updates = state_->NewDBTransaction(); + int refreshes = 0; + vector > invalidation_keys; + + *completed = true; + + for (DB::PrefixIterator iter(snap, DBFormat::user_invalidation_key("")); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + ++refreshes; + int64_t user_id; + if (!DecodeUserInvalidationKey(key, &user_id)) { + LOG("day table: unable to decode user invalidation key: %s", key); + continue; + } + + // Add invalidation key for deletion under mutex. + const int64_t snapshot_isn = FromString(value); + invalidation_keys.push_back(std::make_pair(key.ToString(), snapshot_isn)); + } + + MutexLock l(&mu_); + DeleteInvalidationKeysLocked(invalidation_keys, updates); + CHECK(updates->Commit(false)) << "failed database update"; + + if (refreshes > 0) { + LOG("day table: %d users refreshed in %.3fs", refreshes, timer.Get()); + } + return refreshes; +} + +void DayTable::GarbageCollect() { + DBHandle updates = state_->NewDBTransaction(); + + for (DB::PrefixIterator iter(updates, kDayKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + // Parse the day's timestamp key and delete the day if it + // doesn't match its own canonicalization. + WallTime timestamp; + if (!DecodeDayKey(key, ×tamp) || + timestamp != CanonicalizeTimestamp(timestamp)) { + updates->Delete(key); + } + } + const int day_count = updates->tx_count(); + int last_count = updates->tx_count(); + if (day_count > 0) { + LOG("day table: garbage collected %d days", day_count); + } + for (DB::PrefixIterator iter(updates, kDayEventKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + WallTime timestamp; + int index; + if (!DecodeDayEventKey(key, ×tamp, &index) || + timestamp != CanonicalizeTimestamp(timestamp)) { + updates->Delete(key); + } + } + const int event_count = updates->tx_count() - last_count; + last_count = updates->tx_count(); + if (event_count > 0) { + LOG("day table: garbage collected %d day events", event_count); + } + for (DB::PrefixIterator iter(updates, kDayEpisodeInvalidationKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + WallTime timestamp; + int64_t episode_id; + if (!DecodeDayEpisodeInvalidationKey(key, ×tamp, &episode_id) || + timestamp != CanonicalizeTimestamp(timestamp)) { + updates->Delete(key); + } + } + const int day_episode_count = updates->tx_count() - last_count; + last_count = updates->tx_count(); + if (day_episode_count > 0) { + LOG("day table: garbage collected %d day episode invalidations", day_episode_count); + } + updates->Commit(); +} + +WallTime CanonicalizeTimestamp(WallTime timestamp) { + // Start at minimum offset to avoid negative timestamps. + timestamp = std::max(timestamp, kMinTimestamp); + return CurrentDay(timestamp - kPracticalDayOffset) + kPracticalDayOffset; +} + +string EncodeDayKey(WallTime timestamp) { + string s; + OrderedCodeEncodeVarint32(&s, timestamp); + return DBFormat::day_key(s); +} + +string EncodeDayEventKey(WallTime timestamp, int index) { + string s; + OrderedCodeEncodeVarint32(&s, timestamp); + OrderedCodeEncodeVarint32(&s, index); + return DBFormat::day_event_key(s); +} + +string EncodeDayEpisodeInvalidationKey( + WallTime timestamp, int64_t episode_id) { + string s; + OrderedCodeEncodeVarint32Decreasing(&s, timestamp); + OrderedCodeEncodeVarint64(&s, episode_id); + return DBFormat::day_episode_invalidation_key(s); +} + +string EncodeEpisodeEventKey(int64_t episode_id) { + string s; + OrderedCodeEncodeVarint64(&s, episode_id); + return DBFormat::episode_event_key(s); +} + +string EncodeTimestampAndIdentifier(WallTime timestamp, int64_t identifier) { + string s; + OrderedCodeEncodeVarint32(&s, timestamp); + OrderedCodeEncodeVarint64(&s, identifier); + return s; +} + +string EncodeTrapdoorEventKey(int64_t viewpoint_id, const string& event_key) { + return DBFormat::trapdoor_event_key(viewpoint_id, event_key); +} + +string EncodeTrapdoorKey(int64_t viewpoint_id) { + string s; + OrderedCodeEncodeVarint64(&s, viewpoint_id); + return DBFormat::trapdoor_key(s); +} + +string EncodeUserInvalidationKey(int64_t user_id) { + string s; + OrderedCodeEncodeVarint64(&s, user_id); + return DBFormat::user_invalidation_key(s); +} + +string EncodeViewpointConversationKey(int64_t viewpoint_id) { + string s; + OrderedCodeEncodeVarint64(&s, viewpoint_id); + return DBFormat::viewpoint_conversation_key(s); +} + +string EncodeViewpointInvalidationKey( + WallTime timestamp, int64_t viewpoint_id, int64_t activity_id) { + string s; + OrderedCodeEncodeVarint32Decreasing(&s, timestamp); + OrderedCodeEncodeVarint64(&s, viewpoint_id); + OrderedCodeEncodeVarint64(&s, activity_id); + return DBFormat::viewpoint_invalidation_key(s); +} + +string EncodeViewpointSummaryKey(int64_t viewpoint_id) { + return Format("%s%d", DBFormat::viewpoint_summary_key(), viewpoint_id); +} + +bool DecodeDayKey(Slice key, WallTime* timestamp) { + if (!key.starts_with(kDayKeyPrefix)) { + return false; + } + key.remove_prefix(kDayKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32(&key); + return true; +} + +bool DecodeDayEventKey(Slice key, WallTime* timestamp, int* index) { + if (!key.starts_with(kDayEventKeyPrefix)) { + return false; + } + key.remove_prefix(kDayEventKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32(&key); + *index = OrderedCodeDecodeVarint32(&key); + return true; +} + +bool DecodeDayEpisodeInvalidationKey( + Slice key, WallTime* timestamp, int64_t* episode_id) { + if (!key.starts_with(kDayEpisodeInvalidationKeyPrefix)) { + return false; + } + key.remove_prefix(kDayEpisodeInvalidationKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32Decreasing(&key); + *episode_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeTimestampAndIdentifier(Slice key, WallTime* timestamp, int64_t* identifier) { + *timestamp = OrderedCodeDecodeVarint32(&key); + *identifier = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeTrapdoorKey(Slice key, int64_t* viewpoint_id) { + if (!key.starts_with(kTrapdoorKeyPrefix)) { + return false; + } + key.remove_prefix(kTrapdoorKeyPrefix.size()); + *viewpoint_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeUserInvalidationKey(Slice key, int64_t* user_id) { + if (!key.starts_with(kUserInvalidationKeyPrefix)) { + return false; + } + key.remove_prefix(kUserInvalidationKeyPrefix.size()); + *user_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeViewpointInvalidationKey( + Slice key, WallTime* timestamp, int64_t* viewpoint_id, int64_t* activity_id) { + if (!key.starts_with(kViewpointInvalidationKeyPrefix)) { + return false; + } + key.remove_prefix(kViewpointInvalidationKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32Decreasing(&key); + *viewpoint_id = OrderedCodeDecodeVarint64(&key); + *activity_id = OrderedCodeDecodeVarint64(&key); + return true; +} + + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/DayTable.h b/clients/shared/DayTable.h new file mode 100644 index 0000000..809a2c0 --- /dev/null +++ b/clients/shared/DayTable.h @@ -0,0 +1,912 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_DAY_TABLE_H +#define VIEWFINDER_DAY_TABLE_H + +#import +#import +#import "ActivityTable.h" +#import "Callback.h" +#import "DayMetadata.pb.h" +#import "DB.h" +#import "EpisodeTable.h" +#import "Location.pb.h" +#import "Mutex.h" +#import "Placemark.pb.h" +#import "ScopedHandle.h" +#import "ScopedPtr.h" +#import "ViewpointTable.h" +#import "WallTime.h" + +class AppState; +class DayTableEnv; + +enum ActivityThreadType { + THREAD_START, // Start of a thread. + THREAD_PHOTOS, // Addition of photos. + THREAD_END, // Last comment in a thread. + THREAD_POINT, // Continuation of a thread. + THREAD_COMBINE, // Two successive comments combined to look like one. + THREAD_COMBINE_NEW_USER, // Combine but to new user. + THREAD_COMBINE_WITH_TIME, // Two successive comments combined, but with time appended. + THREAD_COMBINE_NEW_USER_WITH_TIME, // Combine requiring a time, but to new user. + THREAD_COMBINE_END, // Combined, but final comment. + THREAD_COMBINE_END_WITH_TIME, // Combine, but final comment with time prepended. + THREAD_NONE, +}; + +enum ActivityUpdateType { + UPDATE_SINGLE, // Lone update. + UPDATE_START, // First update in a series. + UPDATE_COMBINE, // Middle update in a series. + UPDATE_END, // Final update in a series. +}; + +// DEPRECATED: remove when conversations stop using the episode layout. +enum EpisodeLayoutType { + EVENT_SUMMARY_LAYOUT, + CONVERSATION_LAYOUT, + EVENT_EPISODE_LAYOUT, +}; + +enum PhotoLayoutType { + SUMMARY_COLLAPSED_LAYOUT, + SUMMARY_EXPANDED_LAYOUT, + FULL_EVENT_LAYOUT, +}; + +bool IsThreadTypeCombine(ActivityThreadType type); + +typedef std::unordered_set PhotoIdSet; + +// The DayTable class provides access to cached summaries of +// episodes and activites over the span of single days. The +// contents are invalidated with any added or modified activity. +// The mapping in the datamodel is from: +// +// -> +// +// There is also a mapping from day timestamp to the most recent +// invalidation sequence number. +// +// , -> <> +// +// Timestamps are canonicalized to day granularity using +// CanonicalizeTimestamp. +// +// DayTable maintains a database snapshot from which it builds all of +// its summaries. The summaries themselves, however, are read and +// written using the read-write database. +// +// DayTable is thread-safe. All other objects are not and should +// be treated as read-only. +class DayTable { + public: + class Day; + class Event; + + // Trapdoor. A viewpoint trapdoor, as the name suggests, provides a + // link to a viewpoint. The metadata includes a summarization of the + // goings-on within the linked viewpoint. + // + // There are three types of trapdoors: inboxes (INBOX), which cover + // the entire history of a viewpoint, and events (EVENT), which give + // detailed information in the event view about how episode(s) were + // shared. + class Trapdoor : public TrapdoorMetadata { + friend class Day; + friend class DayTable; + friend class Event; + friend class ScopedHandle; + + public: + // Returns a formatted timestamp. + string FormatTimestamp(bool shorten) const; + + // Returns a timestamp formatted as delta from current time ("time ago"). + string FormatTimeAgo() const; + + // Returns a formatted list of contributors, without regard for + // whether any has contributed new content. "contributor_mask" is + // a bitwise or of one or more of the values described in + // DayMetadata.proto for DayContributor::ContributorType. Specify + // 0 (default) for all types. + string FormatContributors(bool shorten, int contributor_mask = 0) const; + + // Formats photo & comment counts. + string FormatPhotoCount() const; + string FormatCommentCount() const; + + // Returns true if should display in summary. + bool DisplayInSummary() const; + + // Returns true if should display in inbox. + bool DisplayInInbox() const; + + // Returns true if should display in event view. + bool DisplayInEvent() const; + + // Returns true if there is no content in the trapdoor. + bool IsEmpty() const; + + ViewpointHandle GetViewpoint() const; + + private: + Trapdoor(AppState* state, const DBHandle& db); + + // Initialize an INBOX type trapdoor from a pre-built viewpoint + // summary. + void InitFromViewpointSummary(const ViewpointSummaryMetadata& vs); + + // Adds the specified share activity to the summary. The share + // specifically is adding the provided CachedEpisode, + // "ce". Updates the first and last timestamps and contributors. + void AddSharedEpisode(const ActivityHandle& ah, const CachedEpisode* ce); + + // Canonicalizes internal state by calling SamplePhotos, + // CanonicalizeContributors and CanonicalizeTitle to flesh out + // metadata. + const TrapdoorMetadata& Canonicalize(); + + // Free up resources used to build the trapdoor. + void Cleanup(); + + // Samples photos from the available shares in round robin + // fashion. + void SamplePhotos(); + + // Sets the cover photo information, if at least one photo is available. + void MaybeSetCoverPhoto(); + + // Sorts active contributors by max_update_seq. + void CanonicalizeContributors(); + + // Queries viewpoint for title information. + void CanonicalizeTitle(); + + void Ref() { refcount_.Ref(); } + void Unref() { if (refcount_.Unref()) delete this; } + + private: + AppState* state_; + DBHandle db_; + AtomicRefCount refcount_; + ViewpointHandle viewpoint_; + // Map user_id => max_update_seq. Value is a floating point number + // to allow a slightly larger update sequence for the actual + // contributor, so in the case of adding followers to a viewpoint, + // the inviter sorts before invitees. + std::unordered_map contributors_; + // For prospective users who do not yet have a user id. + std::unordered_map contributors_by_identity_; + // For sampling. These are only set for episodes which are + // contemporaneous with the day of the trapdoor. For an INBOX + // trapdoor, this means the day of the most recent activity. + // The bool in the pair is true if this event is new, and should + // count towards the new photo count. + vector > episodes_; + }; + typedef ScopedHandle TrapdoorHandle; + + // Event. An event is a collection of episodes in the user's + // personal library. Each event contains filtered episodes ordered + // from least recent to most recent, and maintains an invariant that + // all episodes in a group be within a threshold distance and + // threshold time of each other. Episodes without any location on + // any photos are grouped into a separate "location-less" group. The + // groups themselves are also ordered from least to most recent + // within the day, with the earliest timestamp of any episode within + // the group being used as the sort key. + // + // A canonical location and associated placemark is determined by + // finding the episode from amongst the group with location closest + // to the group centroid and using its placemark. + class Event : public EventMetadata { + friend class Day; + friend class DayTable; + friend class ScopedHandle; + + public: + // Returns a formatted title (combines related convo title & location). + string FormatTitle(bool shorten) const; + + // Returns formatted location. + string FormatLocation(bool shorten, bool uppercase = false) const; + + // Returns a formatted representation of related convos (trapdoors). + string FormatRelatedConvos(bool shorten) const; + + // Returns a formatted timestamp. + string FormatTimestamp(bool shorten) const; + + // Returns the formatted time span of the event from earliest + // photo timestamp to latest. + string FormatTimeRange(bool shorten) const; + + // Returns a formatted list of contributors. + string FormatContributors(bool shorten) const; + + // Formats photo count. + string FormatPhotoCount() const; + + // Returns a vector of event trapdoors. + const vector& trapdoors() const { return trapdoors_; } + + // Returns true if there is no content in the event. + bool IsEmpty() const; + + private: + Event(AppState* state, const DBHandle& db); + + // Returns true if "timestamp" falls between the anchor episode's + // earliest and latest timestamps with a margin of "margin_secs" + // at each bookend time. + bool WithinTimeRange( + const CachedEpisode* anchor, WallTime timestamp, double margin_secs); + + // Returns true if the event already contains any of the photos + // which are part of the specified episode. + bool ContainsPhotosFromEpisode(const CachedEpisode* ce); + + // Returns true if the episode can be added to this event based + // on geo-temporal proximity to the "anchor" event. + bool CanAddEpisode( + const CachedEpisode* anchor, const CachedEpisode* ce, float threshold_ratio); + + // Adds the episode to the event. + void AddEpisode(const CachedEpisode* ce); + + // Canonicalizes internal state by calling CanonicalizeEpisodes + // and CanonicalizeLocation to flesh out metadata. + const EventMetadata& Canonicalize(); + + // Cleanup the resources used to build the event. + void Cleanup(); + + // Sorts the episodes by timestamp in descending order, determines + // the first and last timestamps, the set of contributors sorted + // by greatest to least contribution. + void CanonicalizeEpisodes(); + + // Determines canonical location and placemark. + void CanonicalizeLocation(); + + // Processes all of the EVENT trapdoors under this event. + void CanonicalizeTrapdoors(); + + void Ref() { refcount_.Ref(); } + void Unref() { if (refcount_.Unref()) delete this; } + + private: + AppState* state_; + DBHandle db_; + AtomicRefCount refcount_; + vector episodes_; + std::unordered_set photo_ids_; + vector trapdoors_; + + static const double kHomeVsAwayThresholdMeters; + static const double kHomeThresholdMeters; + static const double kAwayThresholdMeters; + static const double kHomeThresholdSeconds; + static const double kAwayThresholdSeconds; + + static const double kExtendThresholdRatio; + static const double kExoticThresholdMeters; + }; + typedef ScopedHandle EventHandle; + + // The Day class provides an interface into the goings-on for a single + // day. The timestamps which signify the beginning and end of a day are + // not defined by 12 midnight to 12 midnight, but from an arbitrary + // "real-world-offset" into a typical day, defaulting to 4 hours and + // 30 minutes--meaning that days start at 4:30a and end at 4:30a the + // following day. + // + // Day objects may be individually addressed using a timestamp "ts". The + // range of assets displayed during that day are determined by using + // the range: + // + // [WallTime::CurrentDay(ts) + the "real-world-offset", +24h:00m) + class Day { + friend class DayTable; + + public: + WallTime timestamp() const { return metadata_.timestamp(); } + + // Returns true if the episode handle represents a valid + // episode. This may not be the case where an episode has only + // been partially loaded (or not yet), but is referenced by an + // activity. + static bool IsEpisodeFullyLoaded(const EpisodeHandle& eh); + + // Initializes a CachedEpisode metadata from an episode handle. + // If not NULL, photo_id_filter is applied against the photos + // contained in the episode to determine inclusion. + static void InitCachedEpisode( + AppState* state, const EpisodeHandle& eh, CachedEpisode* ce, const DBHandle& db, + const std::unordered_set* photo_id_filter = NULL); + + private: + Day(AppState* state, WallTime timestamp, const DBHandle& db); + + // Loads the cached day metadata to be augmented by activities + // or episodes as needed. + bool Load(); + + // Saves derived events to database. Called from Rebuild() or + // UpdateEpisodes(). + void Save(vector* events, const DBHandle& updates); + + // Rebuilds day from scratch by iterating over episode + // table. Creates EVENT trapdoors for all viewpoints which have + // episodes which occurred on this day. Creates EVENT trapdoors + // for shares from photos within an event. Sorts vectors according + // to applicable orderings and stores in db. + void Rebuild(vector* events, const DBHandle& updates); + + // Updates day metadata by refreshing the specified vector of episodes. + void UpdateEpisodes(const vector& episode_ids, + vector* events, const DBHandle& updates); + + // Segments cached episodes array into events. + void SegmentEvents(vector* events); + + + // Creates an EVENT trapdoor to display the disposition of sharing + // for photos within an event view. + void CreateEventTrapdoor(vector* events, const CachedEpisode* ce, + const ViewpointHandle& vh, int ev_index); + + private: + AppState* state_; + DBHandle db_; + DayMetadata metadata_; + }; + + friend class Day; + + // Summary provides aggregate details and by-index querying of + // summary rows across all days in the user's history. Summary rows + // can be arbitrary types of information including events, fully- + // expanded events, and conversations. See the subclasses of Summary + // for examples. + class Summary { + friend class DayTable; + + public: + enum SummaryType { + EVENTS, + FULL_EVENTS, + CONVERSATIONS, + UNVIEWED_CONVERSATIONS, + }; + + public: + Summary(DayTable* day_table); + virtual ~Summary(); + + // Member accessors. + virtual int photo_count() const { return summary_.photo_count(); } + virtual int row_count() const { return summary_.rows_size(); } + virtual int total_height() const { return summary_.total_height(); } + + // Get summary row by "row_index". + virtual bool GetSummaryRow(int row_index, SummaryRow* row) const; + + void Ref() { refcount_.Ref(); } + void Unref() { if (refcount_.Unref()) delete this; } + + protected: + // Get summary row index by day "timestamp" and "identifier". + int GetSummaryRowIndex(WallTime timestamp, int64_t identifier) const; + + // Adds summary rows comprising a full day. + void AddDayRows(WallTime timestamp, const vector& rows); + + // Removes all summary rows for the specified day. + void RemoveDayRows(WallTime timestamp); + + // Adds a single summary row. + void AddRow(const SummaryRow& row); + + // Removes the summary row at "index". + void RemoveRow(int index); + + // Loads summary information. Returns true if successfully loaded. + bool Load(const string& key, const DBHandle& db); + + // Saves the summary information. + void Save(const string& key, const DBHandle& updates); + + // Sets row positions and gets set of holidays by day timestamp. + void Normalize(); + + // Computes a weight contribution by normalizing "value" by "max". + // If "max" is 0, returns 0. If "log_scale" is true, computes the + // weight contribution after converting both "value" and "max" to + // a logarithmic scale. + float ComputeWeight(float value, float max, bool log_scale) const; + + // Normalizes the weight of a summary row. Stores weight in row->weight(). + void NormalizeRowWeight(SummaryRow* row, bool is_holiday) const; + + // Returns a height prefix for computing absolute positions. + virtual float height_prefix() const { return 0; } + + // Returns a height suffix for computing absolute positions. + virtual float height_suffix() const { return 0; } + + AppState* state() const { return day_table_->state_; } + DayTableEnv* env() const { return day_table_->env_.get(); } + + protected: + DayTable* const day_table_; + DBHandle db_; + SummaryMetadata summary_; + AtomicRefCount refcount_; + + // For normalizing weights. + int photo_count_max_; + int comment_count_max_; + int contributor_count_max_; + int share_count_max_; + double distance_max_; + + // Relative importance of various contributions to row weight. The + // weights are used to rank order rows. The ordering prioritizes the + // display of rows in the viewfinder when there are too many to fit. + static const float kPhotoVolumeWeightFactor; + static const float kCommentVolumeWeightFactor; + static const float kContributorWeightFactor; + static const float kShareWeightFactor; + static const float kDistanceWeightFactor; + static const float kUnviewedWeightBonus; + }; + + friend class Summary; + + class EventSummary : public Summary { + friend class DayTable; + + public: + EventSummary(DayTable* day_table); + + // Get event summary row index by episode id. Returns -1 if the + // episode could not be located. + int GetEpisodeRowIndex(int64_t episode_id) const; + + // Get the list of event summary row indexes with trapdoors pointing to the given episode id. + void GetViewpointRowIndexes(int64_t viewpoint_id, vector* row_indexes) const; + + // Writes episode ids for fast lookup and calls through to base class. + void UpdateDay(WallTime timestamp, + const vector& events, const DBHandle& updates); + + bool Load(const DBHandle& db); + void Save(const DBHandle& updates); + + protected: + // Add 4pt prefix and suffix to height of summary. + float height_prefix() const { return 4; } + float height_suffix() const { return 4; } + }; + typedef ScopedHandle EventSummaryHandle; + + + class ConversationSummary : public Summary { + friend class DayTable; + + public: + // Accessor for unviewed inbox count. + int unviewed_inbox_count() const { return summary_.unviewed_count(); } + + // Get event summary row index by viewpoint id. + int GetViewpointRowIndex(int64_t viewpoint_id) const; + + // Writes viewpoint ids for fast lookup and updates/adds a summary. + void UpdateTrapdoor( + const Trapdoor& trap, const DBHandle& updates); + + // Removes a trapdoor from the summary. + void RemoveTrapdoor(int64_t viewpoint_id, const DBHandle& updates); + + bool Load(const DBHandle& db); + void Save(const DBHandle& updates); + + protected: + // Add 4pt prefix and suffix to height of summary. + float height_prefix() const { return 4; } + float height_suffix() const { return 4; } + + private: + ConversationSummary(DayTable* day_table); + + void SanityCheck(const DBHandle& db); + void SanityCheckRemoved(int64_t viewpoint_id); + }; + typedef ScopedHandle ConversationSummaryHandle; + + + class UnviewedConversationSummary : public Summary { + friend class DayTable; + + public: + void UpdateTrapdoor( + const Trapdoor& trap, const DBHandle& updates); + + void RemoveTrapdoor(int64_t viewpoint_id, const DBHandle& updates); + + bool Load(const DBHandle& db); + void Save(const DBHandle& updates); + + private: + UnviewedConversationSummary(DayTable* day_table); + }; + typedef ScopedHandle UnviewedConversationSummaryHandle; + + + class FullEventSummary : public Summary { + friend class DayTable; + + public: + // Get event summary row index by episode id. Returns -1 if the + // episode could not be located. + int GetEpisodeRowIndex(int64_t episode_id) const; + + // Get the list of event summary row indexes with trapdoors pointing to the given episode id. + void GetViewpointRowIndexes(int64_t viewpoint_id, vector* row_indexes) const; + + void UpdateDay(WallTime timestamp, + const vector& events, const DBHandle& updates); + + bool Load(const DBHandle& db); + void Save(const DBHandle& updates); + + protected: + virtual float height_suffix() const; + + private: + FullEventSummary(DayTable* day_table); + }; + typedef ScopedHandle FullEventSummaryHandle; + + + // Summary describes the skeleton of a viewpoint's activities for + // efficiently displaying the visible portion. + class ViewpointSummary : public ViewpointSummaryMetadata { + public: + ViewpointSummary(DayTable* day_table, const DBHandle& db); + ~ViewpointSummary(); + + // Loads summary information. Returns true if summary information could + // be loaded; otherwise false indicates the summary should be built from + // scratch. + bool Load(int64_t viewpoint_id); + + // Saves the summary information. Computes and writes trapdoor metadata + // to the database. Returns the result in "*trap". + void Save(const DBHandle& updates, Trapdoor* trap); + + // Rebuilds summary from scratch. + void Rebuild(const ViewpointHandle& vh); + + // Incrementally updates summary by adding or updating the specified + // activity "ah". Merges the existing array of activity rows with those + // specified in "ah_vec", ignoring existing rows being replaced. + // REQUIRES: ah_vec is sorted by activity timestamps. + void UpdateActivities( + const ViewpointHandle& vh, const vector& ah_vec); + + // Computes the final row heights, but only when run on the main thread + // since some of the row heights depend on UIKit functionality that is not + // available on non-main threads. + void UpdateRowHeights(const ViewpointHandle& vh); + + // Computes the final row positions based upon the current row heights. + void UpdateRowPositions(); + + // Deletes viewpoint summary. + static void Delete(int64_t id, const DBHandle& updates); + + // Returns true if there are no photos and no comments. + bool IsEmpty() const; + + float total_height() const { return total_height_; } + + void Ref() { refcount_.Ref(); } + void Unref() { if (refcount_.Unref()) delete this; } + + private: + // Set positions (based on cumulative row heights), timestamp + // range, asset counts, and sort contributors by recent activity. + void Normalize(const ViewpointHandle& vh); + + // Loads and returns the specified activity (unless "activity_id" == -1). + ActivityHandle LoadActivity(int64_t activity_id); + + // Appends header row including cover photo and viewpoint title & + // followers. + void AppendHeaderRow( + const ViewpointHandle& vh, const ActivityHandle& ah); + + // Appends activity row(s) for the specified activity (possibly taking + // previous and next activity into account). + void AppendActivityRows( + const ViewpointHandle& vh, const ActivityHandle& ah, + const ActivityHandle& prev_ah, const ActivityHandle& next_ah, + std::unordered_set* unique_ids); + + AppState* state() const { return day_table_->state_; } + DayTableEnv* env() const { return day_table_->env_.get(); } + + private: + DayTable* const day_table_; + DBHandle db_; + AtomicRefCount refcount_; + float total_height_; + }; + + friend class ViewpointSummary; + typedef ScopedHandle ViewpointSummaryHandle; + + // A snapshot of day table state, including all summary handles: + // (events, full events, conversations, unviewed conversations). Use + // this object to access cached day table metadata across the entire + // spectrum at a single slice in time. + // + // NOTE: be careful to release these objects frequently as live + // references to leveldb snapshots causes the underlying sstables + // not to merge. + class Snapshot { + public: + Snapshot(AppState* state, const DBHandle& snapshot_db); + ~Snapshot(); + + // Accessor for the underlying database snapshot. + const DBHandle& db() const { return snapshot_db_; } + + // Fetches the day table summary information for events. + const EventSummaryHandle& events() const { + return events_; + } + + // Fetches the day table summary information for fully expanded events. + const FullEventSummaryHandle& full_events() const { + return full_events_; + } + + // Fetches the day table summary information for conversations. + const ConversationSummaryHandle& conversations() const { + return conversations_; + } + + // Fetches the day table summary information for conversations. + const UnviewedConversationSummaryHandle& unviewed_conversations() const { + return unviewed_conversations_; + } + + // Fetches the viepwoint summary for the specified viewpoint. + ViewpointSummaryHandle LoadViewpointSummary(int64_t viewpoint_id) const; + + // Loads the event specified by the timestamp / index combination. + EventHandle LoadEvent(WallTime timestamp, int index); + + // Loads the trapdoor specified by viewpoint id. + TrapdoorHandle LoadTrapdoor(int64_t viewpoint_id); + + // Returns a trapdoor for the specified viewpoint & photo combination. + TrapdoorHandle LoadPhotoTrapdoor(int64_t viewpoint_id, int64_t photo_id); + + void Ref() { refcount_.Ref(); } + void Unref() { if (refcount_.Unref()) delete this; } + + private: + AppState* state_; + DBHandle snapshot_db_; + AtomicRefCount refcount_; + EventSummaryHandle events_; + FullEventSummaryHandle full_events_; + ConversationSummaryHandle conversations_; + UnviewedConversationSummaryHandle unviewed_conversations_; + }; + typedef ScopedHandle SnapshotHandle; + + public: + DayTable(AppState* state, DayTableEnv* env); + ~DayTable(); + + // Initialize the day table; verifies day table format version and + // timezone and starts garbage collection cycle. Returns true if + // the day table's contents were reset for a full refresh. + bool Initialize(bool force_reset); + + bool initialized() const; + + // Fetches the day table summary information as a snapshot in time. + const SnapshotHandle& GetSnapshot(int* epoch); + + // Invalidates an activity; this affects both the day upon which the + // activity occurred as well as the viewpoint to which the activity + // belongs. + void InvalidateActivity(const ActivityHandle& ah, const DBHandle& updates); + + // Invalidate any cached day metadata for the specified timestamp. + void InvalidateDay(WallTime timestamp, const DBHandle& updates); + + // Invalidates an episode; this affects the day upon which the + // episode occurred. + void InvalidateEpisode(const EpisodeHandle& eh, const DBHandle& updates); + + // Invalidates viewpoint metadata. + void InvalidateViewpoint(const ViewpointHandle& vh, const DBHandle& updates); + + // Invalidates user contact metadata. + void InvalidateUser(int64_t user_id, const DBHandle& updates); + + // Invalidates current snapshot and invokes all update callbacks. + void InvalidateSnapshot(); + + // Pause/resume all/event refreshes. If "callback" is not NULL, it is invoked + // after the first round of refreshes is complete. + void PauseAllRefreshes(); + void PauseEventRefreshes(); + void ResumeAllRefreshes(Callback callback = nullptr); + void ResumeEventRefreshes(); + + // Returns true if there are refreshes still pending; false otherwise. + bool refreshing() const; + + AppState* state() const { return state_; } + + // Callers can register to be notified when day metadata has been + // refreshed and a new snapshot epoch has arrived. + CallbackSet* update() { return &update_; } + + // TODO(spencer): this is completely temporary; needs to be + // normalized to handle a specific user's locale. Probably + // going to want a "CalendarTable" and will need a hook into + // network manager to query events from the server. + // + // Returns true if the specified timestamp falls on a holiday. + // "*s" is set to the holiday name. + bool IsHoliday(WallTime timestamp, string* s); + + private: + void InvalidateDayLocked(WallTime timestamp, const DBHandle& updates); + void InvalidateViewpointLocked(const ViewpointHandle& vh, const DBHandle& updates); + + // Invalidates current snapshot and invokes all update callbacks. + void InvalidateSnapshotLocked(); + + // Loads summary information. Called from InvalidateSnapshotLocked. + void LoadSummariesLocked(const DBHandle& db); + + // Returns whether the system time zone has changed from the + // timezone persisted to the database. + bool ShouldUpdateTimezone() const; + + // Updates the timezone used for computing local times. + void UpdateTimezone(const DBHandle& updates); + + // Clears all cached day metadata and prepares for a complete rebuild. + void ResetLocked(const DBHandle& updates); + void DeleteDayTablesLocked(const DBHandle& updates); + void InvalidateAllDaysLocked(const DBHandle& updates); + void InvalidateAllViewpointsLocked(const DBHandle& updates); + + // Deletes invalidation keys under mutex mu_, verifying the + // invalidation sequence number of each before removing the key. + void DeleteInvalidationKeysLocked( + const vector >& invalidation_keys, + const DBHandle& updates); + + // Dispatch a low-priority thread to refresh invalidating day + // metadata contents as needed. If a refresh is already underway, + // returns immediately as a no-op. + void MaybeRefresh(Callback callback = nullptr); + + // Refresh any days for which metadata has been invalidated. Sets + // *completed to true if the refresh processed all invalidated + // days. Returns the number of days refreshed. + int RefreshDayEpisodes(bool* completed); + + // Refresh any viewpoints for which activities have been + // invalidated. Sets *completed to true if refresh processed + // all pending invalidations. Returns the number of viewpoints + // which were refreshed. + int RefreshViewpoints(bool* completed); + + // Refresh any updated users. This is mostly a no-op at the moment as + // User contact info is utilized on the fly when displaying instead of + // being baked into the cached display metadata. + int RefreshUsers(bool* completed); + + // Garbage collect any days which were created under different + // assumptions as to when a day "begins". This will happen if the + // user's timezone changes or if we (or maybe even the user) decides + // to modify the "practical day offset". + // + // Called on startup to run in a background thread. + // TODO(spencer): unittest this. + void GarbageCollect(); + + private: + AppState* state_; + ScopedPtr env_; + mutable Mutex mu_; + int epoch_; + SnapshotHandle snapshot_; + CallbackSet update_; + bool initialized_; + bool all_refreshes_paused_; + bool event_refreshes_paused_; + bool refreshing_; + int64_t invalidation_seq_no_; + string timezone_; + std::unordered_map holiday_timestamps_; +}; + +typedef DayTable::Event Event; +typedef DayTable::EventHandle EventHandle; +typedef DayTable::Trapdoor Trapdoor; +typedef DayTable::TrapdoorHandle TrapdoorHandle; + +// DayTableEnv is the interface the day table uses for querying the UI +// about the size of UI elements. +class DayTableEnv { + public: + virtual ~DayTableEnv() { } + + virtual float GetSummaryEventHeight( + const Event& ev, const DBHandle& db) = 0; + virtual float GetFullEventHeight( + const Event& ev, const DBHandle& db) = 0; + virtual float GetInboxCardHeight( + const Trapdoor& trap) = 0; + virtual float GetConversationHeaderHeight( + const ViewpointHandle& vh, int64_t cover_photo_id) = 0; + virtual float GetConversationActivityHeight( + const ViewpointHandle& vh, const ActivityHandle& ah, + int64_t reply_to_photo_id, ActivityThreadType thread_type, + const DBHandle& db) = 0; + virtual float GetConversationUpdateHeight( + const ViewpointHandle& vh, const ActivityHandle& ah, + ActivityUpdateType update_type, const DBHandle& db) = 0; + virtual float GetShareActivityPhotosRowHeight( + EpisodeLayoutType layout_type, const vector& photos, + const vector& episodes, + const DBHandle& db) = 0; + + virtual float full_event_summary_height_suffix() = 0; +}; + +WallTime CanonicalizeTimestamp(WallTime timestamp); + +string EncodeDayKey(WallTime timestamp); +string EncodeDayEventKey(WallTime timestamp, int index); +string EncodeDayEpisodeInvalidationKey(WallTime timestamp, int64_t episode_id); +string EncodeEpisodeEventKey(int64_t episode_id); +string EncodeTimestampAndIdentifier(WallTime timestamp, int64_t identifier); +string EncodeTrapdoorEventKey(int64_t viewpoint_id, const string& event_key); +string EncodeTrapdoorKey(int64_t viewpoint_id); +string EncodeUserInvalidationKey(int64_t user_id); +string EncodeViewpointConversationKey(int64_t viewpoint_id); +string EncodeViewpointInvalidationKey( + WallTime timestamp, int64_t viewpoint_id, int64_t activity_id); +string EncodeViewpointSummaryKey(int64_t viewpoint_id); +bool DecodeDayKey(Slice key, WallTime* timestamp); +bool DecodeDayEventKey(Slice key, WallTime* timestamp, int* index); +bool DecodeDayEpisodeInvalidationKey(Slice key, WallTime* timestamp, int64_t* episode_id); +bool DecodeTimestampAndIdentifier(Slice key, WallTime* timestamp, int64_t* identifier); +bool DecodeTrapdoorKey(Slice key, int64_t* viewpoint_id); +bool DecodeUserInvalidationKey(Slice key, int64_t* user_id); +bool DecodeViewpointInvalidationKey( + Slice key, WallTime* timestamp, int64_t* viewpoint_id, int64_t* activity_id); + +#endif // VIEWFINDER_DAY_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Defines.h b/clients/shared/Defines.h new file mode 100644 index 0000000..bd2d37f --- /dev/null +++ b/clients/shared/Defines.h @@ -0,0 +1,16 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_DEFINES_H +#define VIEWFINDER_DEFINES_H + +// Only include DeveloperDefines.h if this is not an ad-hoc/app-store build. +#ifdef DEVELOPMENT + #ifndef FLYMAKE + #import "DeveloperDefines.h" + #endif // !FLYMAKE +#else // !DEVELOPMENT +#define PRODUCTION +#endif // !DEVELOPMENT + +#endif // VIEWFINDER_DEFINES_H diff --git a/clients/shared/DigestUtils.android.cc b/clients/shared/DigestUtils.android.cc new file mode 100644 index 0000000..521d529 --- /dev/null +++ b/clients/shared/DigestUtils.android.cc @@ -0,0 +1,375 @@ +#import +#import +#import +#import "DigestUtils.h" +#import "StringUtils.h" + +namespace { + +/********************************************************************* +* Filename: md5.c +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Implementation of the MD5 hashing algorithm. + Algorithm specification can be found here: + * http://tools.ietf.org/html/rfc1321 + This implementation uses little endian byte order. +*********************************************************************/ + +/****************************** MACROS ******************************/ +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) + +#define F(x,y,z) ((x & y) | (~x & z)) +#define G(x,y,z) ((x & z) | (y & ~z)) +#define H(x,y,z) (x ^ y ^ z) +#define I(x,y,z) (y ^ (x | ~z)) + +#define FF(a,b,c,d,m,s,t) { a += F(b,c,d) + m + t; \ + a = b + ROTLEFT(a,s); } +#define GG(a,b,c,d,m,s,t) { a += G(b,c,d) + m + t; \ + a = b + ROTLEFT(a,s); } +#define HH(a,b,c,d,m,s,t) { a += H(b,c,d) + m + t; \ + a = b + ROTLEFT(a,s); } +#define II(a,b,c,d,m,s,t) { a += I(b,c,d) + m + t; \ + a = b + ROTLEFT(a,s); } + +/*********************** FUNCTION DEFINITIONS ***********************/ +void md5_transform(MD5_CTX* ctx, const uint8_t* data) { + uint32_t a, b, c, d, m[16], i, j; + + // MD5 specifies big endian byte order, but this implementation assumes a little + // endian byte order CPU. Reverse all the bytes upon input, and re-reverse them + // on output (in md5_final()). + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j]) + (data[j + 1] << 8) + (data[j + 2] << 16) + (data[j + 3] << 24); + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + + FF(a,b,c,d,m[0], 7,0xd76aa478); + FF(d,a,b,c,m[1], 12,0xe8c7b756); + FF(c,d,a,b,m[2], 17,0x242070db); + FF(b,c,d,a,m[3], 22,0xc1bdceee); + FF(a,b,c,d,m[4], 7,0xf57c0faf); + FF(d,a,b,c,m[5], 12,0x4787c62a); + FF(c,d,a,b,m[6], 17,0xa8304613); + FF(b,c,d,a,m[7], 22,0xfd469501); + FF(a,b,c,d,m[8], 7,0x698098d8); + FF(d,a,b,c,m[9], 12,0x8b44f7af); + FF(c,d,a,b,m[10],17,0xffff5bb1); + FF(b,c,d,a,m[11],22,0x895cd7be); + FF(a,b,c,d,m[12], 7,0x6b901122); + FF(d,a,b,c,m[13],12,0xfd987193); + FF(c,d,a,b,m[14],17,0xa679438e); + FF(b,c,d,a,m[15],22,0x49b40821); + + GG(a,b,c,d,m[1], 5,0xf61e2562); + GG(d,a,b,c,m[6], 9,0xc040b340); + GG(c,d,a,b,m[11],14,0x265e5a51); + GG(b,c,d,a,m[0], 20,0xe9b6c7aa); + GG(a,b,c,d,m[5], 5,0xd62f105d); + GG(d,a,b,c,m[10], 9,0x02441453); + GG(c,d,a,b,m[15],14,0xd8a1e681); + GG(b,c,d,a,m[4], 20,0xe7d3fbc8); + GG(a,b,c,d,m[9], 5,0x21e1cde6); + GG(d,a,b,c,m[14], 9,0xc33707d6); + GG(c,d,a,b,m[3], 14,0xf4d50d87); + GG(b,c,d,a,m[8], 20,0x455a14ed); + GG(a,b,c,d,m[13], 5,0xa9e3e905); + GG(d,a,b,c,m[2], 9,0xfcefa3f8); + GG(c,d,a,b,m[7], 14,0x676f02d9); + GG(b,c,d,a,m[12],20,0x8d2a4c8a); + + HH(a,b,c,d,m[5], 4,0xfffa3942); + HH(d,a,b,c,m[8], 11,0x8771f681); + HH(c,d,a,b,m[11],16,0x6d9d6122); + HH(b,c,d,a,m[14],23,0xfde5380c); + HH(a,b,c,d,m[1], 4,0xa4beea44); + HH(d,a,b,c,m[4], 11,0x4bdecfa9); + HH(c,d,a,b,m[7], 16,0xf6bb4b60); + HH(b,c,d,a,m[10],23,0xbebfbc70); + HH(a,b,c,d,m[13], 4,0x289b7ec6); + HH(d,a,b,c,m[0], 11,0xeaa127fa); + HH(c,d,a,b,m[3], 16,0xd4ef3085); + HH(b,c,d,a,m[6], 23,0x04881d05); + HH(a,b,c,d,m[9], 4,0xd9d4d039); + HH(d,a,b,c,m[12],11,0xe6db99e5); + HH(c,d,a,b,m[15],16,0x1fa27cf8); + HH(b,c,d,a,m[2], 23,0xc4ac5665); + + II(a,b,c,d,m[0], 6,0xf4292244); + II(d,a,b,c,m[7], 10,0x432aff97); + II(c,d,a,b,m[14],15,0xab9423a7); + II(b,c,d,a,m[5], 21,0xfc93a039); + II(a,b,c,d,m[12], 6,0x655b59c3); + II(d,a,b,c,m[3], 10,0x8f0ccc92); + II(c,d,a,b,m[10],15,0xffeff47d); + II(b,c,d,a,m[1], 21,0x85845dd1); + II(a,b,c,d,m[8], 6,0x6fa87e4f); + II(d,a,b,c,m[15],10,0xfe2ce6e0); + II(c,d,a,b,m[6], 15,0xa3014314); + II(b,c,d,a,m[13],21,0x4e0811a1); + II(a,b,c,d,m[4], 6,0xf7537e82); + II(d,a,b,c,m[11],10,0xbd3af235); + II(c,d,a,b,m[2], 15,0x2ad7d2bb); + II(b,c,d,a,m[9], 21,0xeb86d391); + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; +} + +void md5_init(MD5_CTX* ctx) { + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x67452301; + ctx->state[1] = 0xEFCDAB89; + ctx->state[2] = 0x98BADCFE; + ctx->state[3] = 0x10325476; +} + +void md5_update(MD5_CTX* ctx, const uint8_t* data, size_t len) { + size_t i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + md5_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void md5_final(MD5_CTX* ctx, uint8_t* hash) { + size_t i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else if (ctx->datalen >= 56) { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + md5_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[56] = ctx->bitlen; + ctx->data[57] = ctx->bitlen >> 8; + ctx->data[58] = ctx->bitlen >> 16; + ctx->data[59] = ctx->bitlen >> 24; + ctx->data[60] = ctx->bitlen >> 32; + ctx->data[61] = ctx->bitlen >> 40; + ctx->data[62] = ctx->bitlen >> 48; + ctx->data[63] = ctx->bitlen >> 56; + md5_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and MD uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (i * 8)) & 0x000000ff; + } +} + + +/********************************************************************* +* Filename: sha256.c +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Implementation of the SHA-256 hashing algorithm. + SHA-256 is one of the three algorithms in the SHA2 + specification. The others, SHA-384 and SHA-512, are not + offered in this implementation. + Algorithm specification can be found here: + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf + This implementation uses little endian byte order. +*********************************************************************/ + +/****************************** MACROS ******************************/ +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +/**************************** VARIABLES *****************************/ +static const uint32_t k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +/*********************** FUNCTION DEFINITIONS ***********************/ +void sha256_transform(SHA256_CTX* ctx, const uint8_t* data) { + uint32_t a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + e = ctx->state[4]; + f = ctx->state[5]; + g = ctx->state[6]; + h = ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; + t2 = EP0(a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX* ctx) { + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX* ctx, const uint8_t* data, size_t len) { + uint32_t i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX* ctx, uint8_t* hash) { + uint32_t i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = ctx->bitlen; + ctx->data[62] = ctx->bitlen >> 8; + ctx->data[61] = ctx->bitlen >> 16; + ctx->data[60] = ctx->bitlen >> 24; + ctx->data[59] = ctx->bitlen >> 32; + ctx->data[58] = ctx->bitlen >> 40; + ctx->data[57] = ctx->bitlen >> 48; + ctx->data[56] = ctx->bitlen >> 56; + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} + +} // namespace + +////////////////////////////// + +void MD5_Init(MD5_CTX* ctx) { + md5_init(ctx); +} + +void MD5_Update(MD5_CTX* ctx, const void* data, size_t len) { + md5_update(ctx, reinterpret_cast(data), len); +} + +void MD5_Final(MD5_CTX* ctx, uint8_t* digest) { + md5_final(ctx, digest); +} + +string MD5(const Slice& str) { + MD5_CTX ctx; + MD5_Init(&ctx); + MD5_Update(&ctx, str.data(), str.size()); + uint8_t digest[MD5_DIGEST_LENGTH]; + MD5_Final(&ctx, digest); + return BinaryToHex(Slice((const char*)digest, ARRAYSIZE(digest))); +} + +void SHA256_Init(SHA256_CTX* ctx) { + sha256_init(ctx); +} + +void SHA256_Update(SHA256_CTX* ctx, const void* data, size_t len) { + sha256_update(ctx, reinterpret_cast(data), len); +} + +void SHA256_Final(SHA256_CTX* ctx, uint8_t* digest) { + sha256_final(ctx, digest); +} diff --git a/clients/shared/DigestUtils.cc b/clients/shared/DigestUtils.cc new file mode 100644 index 0000000..6cfca55 --- /dev/null +++ b/clients/shared/DigestUtils.cc @@ -0,0 +1,21 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "DigestUtils.h" +#import "Logging.h" + +string MD5HexToBase64(const Slice& str) { + if (str.size() != 32) { + DCHECK_EQ(32, str.size()); + return string(); + } + uint8_t digest[16]; // CC_MD5_DIGEST_LENGTH == 16 + char buf[3] = { '\0', '\0', '\0' }; + for (int i = 0; i < ARRAYSIZE(digest); i++) { + buf[0] = str[2 * i]; + buf[1] = str[2 * i + 1]; + digest[i] = strtol(buf, 0, 16); + } + return Base64Encode(Slice((char*)digest, ARRAYSIZE(digest))); +} diff --git a/clients/shared/DigestUtils.h b/clients/shared/DigestUtils.h new file mode 100644 index 0000000..2ecf320 --- /dev/null +++ b/clients/shared/DigestUtils.h @@ -0,0 +1,75 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_DIGEST_UTILS_H +#define VIEWFINDER_DIGEST_UTILS_H + +#import "Utils.h" + +#if defined(OS_IOS) + +#import + +#define MD5_DIGEST_LENGTH CC_MD5_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH CC_SHA256_DIGEST_LENGTH + +// On IOS, the MD5*/SHA256* routines are simple wrappers around the +// CommonCrypto library. +typedef CC_MD5_CTX MD5_CTX; +inline void MD5_Init(MD5_CTX* ctx) { + CC_MD5_Init(ctx); +} +inline void MD5_Update(MD5_CTX* ctx, const void* data, size_t len) { + CC_MD5_Update(ctx, data, len); +} +inline void MD5_Final(MD5_CTX* ctx, uint8_t* digest) { + CC_MD5_Final(digest, ctx); +} + +typedef CC_SHA256_CTX SHA256_CTX; +inline void SHA256_Init(SHA256_CTX* ctx) { + CC_SHA256_Init(ctx); +} +inline void SHA256_Update(SHA256_CTX* ctx, const void* data, size_t len) { + CC_SHA256_Update(ctx, data, len); +} +inline void SHA256_Final(SHA256_CTX* ctx, uint8_t* digest) { + CC_SHA256_Final(digest, ctx); +} + +#elif defined(OS_ANDROID) + + +#define SHA256_DIGEST_LENGTH 32 +#define MD5_DIGEST_LENGTH 16 + +// On Android, we use our own MD5/SHA256 implementations (provided by the +// internets). +struct MD5_CTX { + uint8_t data[64]; + uint32_t datalen; + uint64_t bitlen; + uint32_t state[4]; +}; + +struct SHA256_CTX { + uint8_t data[64]; + uint32_t datalen; + uint64_t bitlen; + uint32_t state[8]; +}; + +void MD5_Init(MD5_CTX* ctx); +void MD5_Update(MD5_CTX* ctx, const void* data, size_t len); +void MD5_Final(MD5_CTX* ctx, uint8_t* digest); + +void SHA256_Init(SHA256_CTX* ctx); +void SHA256_Update(SHA256_CTX* ctx, const void* data, size_t len); +void SHA256_Final(SHA256_CTX* ctx, uint8_t* digest); + +#endif // defined(OS_ANDROID) + +string MD5(const Slice& str); +string MD5HexToBase64(const Slice& str); + +#endif // VIEWFINDER_DIGEST_UTILS_H diff --git a/clients/shared/DigestUtils.ios.mm b/clients/shared/DigestUtils.ios.mm new file mode 100644 index 0000000..9777edf --- /dev/null +++ b/clients/shared/DigestUtils.ios.mm @@ -0,0 +1,12 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "DigestUtils.h" +#import "StringUtils.h" + +string MD5(const Slice& str) { + uint8_t digest[CC_MD5_DIGEST_LENGTH]; + CC_MD5(str.data(), str.size(), digest); + return BinaryToHex(Slice((const char*)digest, ARRAYSIZE(digest))); +} diff --git a/clients/shared/EpisodeMetadata.proto b/clients/shared/EpisodeMetadata.proto new file mode 100644 index 0000000..b068337 --- /dev/null +++ b/clients/shared/EpisodeMetadata.proto @@ -0,0 +1,38 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +import "ContentIds.proto"; +import "ContactMetadata.proto"; +import "Location.proto"; +import "Placemark.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "EpisodeMetadataPB"; + +message EpisodeMetadata { + optional EpisodeId id = 1; + optional EpisodeId parent_id = 2; + optional ViewpointId viewpoint_id = 3; + optional int64 user_id = 4; + optional int64 sharing_user_id = 5; + optional double timestamp = 6; + optional double publish_timestamp = 12; + optional string title = 7; + optional string description = 8; + optional string name = 11; + + // Bits indicating labels on the user-episode relation. + // NOTE: add labels here; none as of now. + + // Client-side state. + optional double earliest_photo_timestamp = 30; + optional double latest_photo_timestamp = 31; + + optional bool upload_episode = 32; + + repeated string indexed_terms = 33; + repeated string indexed_location_terms = 34; + + optional Location DEPRECATED_location = 9; + optional Placemark DEPRECATED_placemark = 10; +} diff --git a/clients/shared/EpisodeStats.proto b/clients/shared/EpisodeStats.proto new file mode 100644 index 0000000..6d0a013 --- /dev/null +++ b/clients/shared/EpisodeStats.proto @@ -0,0 +1,13 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "EpisodeStatsPB"; + +message EpisodeStats { + required int32 hidden_photos = 5; + required int32 posted_photos = 1; + required int32 quarantined_photos = 4; + required int32 removed_photos = 2; + required int32 unshared_photos = 3; +} diff --git a/clients/shared/EpisodeTable.cc b/clients/shared/EpisodeTable.cc new file mode 100644 index 0000000..e8e8f16 --- /dev/null +++ b/clients/shared/EpisodeTable.cc @@ -0,0 +1,1564 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "AppState.h" +#import "ContactManager.h" +#import "DayTable.h" +#import "EpisodeTable.h" +#import "FullTextIndex.h" +#import "LazyStaticPtr.h" +#import "LocationUtils.h" +#import "NetworkQueue.h" +#import "PlacemarkHistogram.h" +#import "STLUtils.h" +#import "StringUtils.h" +#import "Timer.h" +#import "WallTime.h" + +namespace { + +const int kEpisodeFSCKVersion = 7; + +const string kEpisodeParentChildKeyPrefix = DBFormat::episode_parent_child_key(string()); +const string kEpisodePhotoKeyPrefix = DBFormat::episode_photo_key(string()); +const string kEpisodeSelectionKeyPrefix = DBFormat::episode_selection_key(string()); +const string kEpisodeStatsKey = DBFormat::metadata_key("episode_stats"); +const string kEpisodeTimestampKeyPrefix = DBFormat::episode_timestamp_key(string()); +const string kPhotoEpisodeKeyPrefix = DBFormat::photo_episode_key(string()); + +const string kEpisodeIndexName = "ep"; +const string kLocationIndexName = "epl"; + +const double kMaxTimeDist = 60 * 60; // 1 hour +const double kMaxLocDist = 10000; // 10 km + +const DBRegisterKeyIntrospect kEpisodeKeyIntrospect( + DBFormat::episode_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kEpisodeServerKeyIntrospect( + DBFormat::episode_server_key(), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kEpisodeParentChildKeyIntrospect( + kEpisodeParentChildKeyPrefix, + [](Slice key) { + int64_t parent_id; + int64_t child_id; + if (!DecodeEpisodeParentChildKey(key, &parent_id, &child_id)) { + return string(); + } + return string(Format("%d/%d", parent_id, child_id)); + }, + [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kEpisodePhotoKeyIntrospect( + kEpisodePhotoKeyPrefix, + [](Slice key) { + int64_t episode_id; + int64_t photo_id; + if (!DecodeEpisodePhotoKey(key, &episode_id, &photo_id)) { + return string(); + } + return string(Format("%d/%d", episode_id, photo_id)); + }, + [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kPhotoEpisodeKeyIntrospect( + kPhotoEpisodeKeyPrefix, [](Slice key) { + int64_t photo_id; + int64_t episode_id; + if (!DecodePhotoEpisodeKey(key, &photo_id, &episode_id)) { + return string(); + } + return string(Format("%d/%d", photo_id, episode_id)); + }, NULL); + +const DBRegisterKeyIntrospect kEpisodeSelectionKeyIntrospect( + kEpisodeSelectionKeyPrefix, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kEpisodeTimestampKeyIntrospect( + kEpisodeTimestampKeyPrefix , [](Slice key) { + WallTime timestamp; + int64_t episode_id; + if (!DecodeEpisodeTimestampKey(key, ×tamp, &episode_id)) { + return string(); + } + return string( + Format("%s/%d", DBIntrospect::timestamp(timestamp), episode_id)); + }, NULL); + +class QueryRewriter : public FullTextQueryVisitor { + public: + QueryRewriter(AppState* state) + : state_(state) { + } + + FullTextQuery* ParseAndRewrite(const Slice& query) { + ScopedPtr parsed_query(FullTextQuery::Parse(query)); + stack_.push_back(Accumulator()); + VisitNode(*parsed_query); + CHECK_EQ(stack_.size(), 1); + CHECK_EQ(stack_[0].size(), 1); + return stack_[0][0]; + } + + void VisitTermNode(const FullTextQueryTermNode& node) { + vector terms; + terms.push_back(new FullTextQueryTermNode(node)); + AddContactTerms(node.term(), false, &terms); + stack_.back().push_back(new FullTextQueryOrNode(terms)); + } + + void VisitPrefixNode(const FullTextQueryPrefixNode& node) { + vector terms; + terms.push_back(new FullTextQueryPrefixNode(node)); + AddContactTerms(node.prefix(), true, &terms); + stack_.back().push_back(new FullTextQueryOrNode(terms)); + } + + void VisitParentNode(const FullTextQueryParentNode& node) { + stack_.push_back(Accumulator()); + VisitChildren(node); + FullTextQuery* new_node; + if (node.type() == FullTextQuery::AND) { + new_node = new FullTextQueryAndNode(stack_.back()); + } else { + new_node = new FullTextQueryOrNode(stack_.back()); + } + stack_.pop_back(); + stack_.back().push_back(new_node); + } + + private: + void AddContactTerms(const string& query, bool prefix, vector* terms) { + // Rewrite each term to its union with any matching contact entries. + vector contact_results; + int options = ContactManager::SORT_BY_NAME | ContactManager::VIEWFINDER_USERS_ONLY; + if (prefix) { + options |= ContactManager::PREFIX_MATCH; + } + state_->contact_manager()->Search(query, &contact_results, NULL, options); + for (const ContactMetadata& c : contact_results) { + // Even if the underlying query is in prefix mode, the resulting tokens are always exact matches. + terms->push_back(new FullTextQueryTermNode(ContactManager::FormatUserToken(c.user_id()))); + } + } + + AppState* state_; + + typedef vector Accumulator; + vector stack_; +}; + +} // namespace + +string EncodeEpisodePhotoKey(int64_t episode_id, int64_t photo_id) { + string s; + OrderedCodeEncodeInt64Pair(&s, episode_id, photo_id); + return DBFormat::episode_photo_key(s); +} + +string EncodePhotoEpisodeKey(int64_t photo_id, int64_t episode_id) { + string s; + OrderedCodeEncodeInt64Pair(&s, photo_id, episode_id); + return DBFormat::photo_episode_key(s); +} + +string EncodeEpisodeTimestampKey(WallTime timestamp, int64_t episode_id) { + string s; + OrderedCodeEncodeVarint32(&s, timestamp); + OrderedCodeEncodeVarint64(&s, episode_id); + return DBFormat::episode_timestamp_key(s); +} + +string EncodeEpisodeParentChildKey(int64_t parent_id, int64_t child_id) { + string s; + OrderedCodeEncodeInt64Pair(&s, parent_id, child_id); + return DBFormat::episode_parent_child_key(s); +} + +bool DecodeEpisodePhotoKey(Slice key, int64_t* episode_id, int64_t* photo_id) { + if (!key.starts_with(kEpisodePhotoKeyPrefix)) { + return false; + } + key.remove_prefix(kEpisodePhotoKeyPrefix.size()); + OrderedCodeDecodeInt64Pair(&key, episode_id, photo_id); + return true; +} + +bool DecodePhotoEpisodeKey(Slice key, int64_t* photo_id, int64_t* episode_id) { + if (!key.starts_with(kPhotoEpisodeKeyPrefix)) { + return false; + } + key.remove_prefix(kPhotoEpisodeKeyPrefix.size()); + OrderedCodeDecodeInt64Pair(&key, photo_id, episode_id); + return true; +} + +bool DecodeEpisodeTimestampKey( + Slice key, WallTime* timestamp, int64_t* episode_id) { + if (!key.starts_with(kEpisodeTimestampKeyPrefix)) { + return false; + } + key.remove_prefix(kEpisodeTimestampKeyPrefix.size()); + *timestamp = OrderedCodeDecodeVarint32(&key); + *episode_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodeEpisodeParentChildKey(Slice key, int64_t* parent_id, int64_t* child_id) { + if (!key.starts_with(kEpisodeParentChildKeyPrefix)) { + return false; + } + key.remove_prefix(kEpisodeParentChildKeyPrefix.size()); + OrderedCodeDecodeInt64Pair(&key, parent_id, child_id); + return true; +} + + +//// +// EpisodeTable_Episode + +EpisodeTable_Episode::EpisodeTable_Episode(AppState* state, const DBHandle& db, int64_t id) + : state_(state), + db_(db), + additions_(0), + hiddens_(0), + quarantines_(0), + removals_(0), + unshares_(0), + have_photo_state_(false), + recompute_timestamp_range_(false), + resolved_location_(false) { + mutable_id()->set_local_id(id); +} + +void EpisodeTable_Episode::MergeFrom(const EpisodeMetadata& m) { + // Some assertions that immutable properties don't change. + if (parent_id().has_server_id() && m.parent_id().has_server_id()) { + DCHECK_EQ(parent_id().server_id(), m.parent_id().server_id()); + } + if (viewpoint_id().has_server_id() && m.viewpoint_id().has_server_id()) { + DCHECK_EQ(viewpoint_id().server_id(), m.viewpoint_id().server_id()); + } + if (has_user_id() && m.has_user_id()) { + DCHECK_EQ(user_id(), m.user_id()); + } + + EpisodeMetadata::MergeFrom(m); +} + +void EpisodeTable_Episode::MergeFrom(const ::google::protobuf::Message&) { + DIE("MergeFrom(Message&) should not be used"); +} + + +int64_t EpisodeTable_Episode::GetDeviceId() const { + if (!id().has_server_id()) { + return state_->device_id(); + } + int64_t device_id = 0; + int64_t dummy_id = 0; + WallTime dummy_timestamp = 0; + DecodeEpisodeId( + id().server_id(), &device_id, &dummy_id, &dummy_timestamp); + return device_id; +} + +int64_t EpisodeTable_Episode::GetUserId() const { + return has_user_id() ? user_id() : state_->user_id(); +} + +void EpisodeTable_Episode::AddPhoto(int64_t photo_id) { + PhotoState* s = NULL; + if (photos_.get()) { + s = FindPtrOrNull(photos_.get(), photo_id); + } + + bool new_photo = false; + if (!s && !have_photo_state_) { + // Optimize the common case where photo state has not been + // initialized. Load only the photo state for the photo being added. + if (!photos_.get()) { + photos_.reset(new PhotoStateMap); + } + const string value = db_->Get( + EncodeEpisodePhotoKey(id().local_id(), photo_id)); + if (!value.empty()) { + s = &(*photos_)[photo_id]; + if (value == EpisodeTable::kHiddenValue) { + *s = HIDDEN; + } else if (value == EpisodeTable::kPostedValue) { + *s = POSTED; + } else if (value == EpisodeTable::kQuarantinedValue) { + *s = QUARANTINED; + } else if (value == EpisodeTable::kRemovedValue) { + *s = REMOVED; + } else if (value == EpisodeTable::kUnsharedValue) { + *s = UNSHARED; + } else { + CHECK(false) << "unexpected value: " << value; + } + } + } + + if (!s) { + new_photo = true; + additions_++; + s = &(*photos_)[photo_id]; + } else if (*s == HIDDEN) { + additions_++; + hiddens_--; + } else if (*s == QUARANTINED) { + additions_++; + quarantines_--; + } else if (*s == REMOVED) { + additions_++; + removals_--; + } else if (*s == UNSHARED) { + additions_++; + unshares_--; + } else { + CHECK_NE(*s, HIDE_PENDING) << "hidden pending"; + CHECK_NE(*s, QUARANTINE_PENDING) << "quarantine pending"; + CHECK_NE(*s, REMOVE_PENDING) << "remove pending"; + CHECK_NE(*s, UNSHARE_PENDING) << "unshare pending"; + } + *s = POST_PENDING; + + // When a new photo is added we can incrementally update the timestamp + // range. Otherwise, we have to recompute the timestamp range from scratch. + if (!new_photo) { + recompute_timestamp_range_ = true; + } +} + +void EpisodeTable_Episode::HidePhoto(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + PhotoState* s = FindPtrOrNull(photos_.get(), photo_id); + if (s) { + if (*s == POSTED) { + additions_--; + hiddens_++; + *s = HIDE_PENDING; + } else if (*s == QUARANTINED) { + quarantines_--; + hiddens_++; + *s = HIDE_PENDING; + } else if (*s == REMOVED) { + removals_--; + hiddens_++; + *s = HIDE_PENDING; + } else { + CHECK_NE(*s, POST_PENDING) << "post pending"; + CHECK_NE(*s, QUARANTINE_PENDING) << "quarantine pending"; + CHECK_NE(*s, REMOVE_PENDING) << "remove pending"; + CHECK_NE(*s, UNSHARE_PENDING) << "unshare pending"; + } + } else { + // We've queried a photo as part of this episode which has been + // removed. We still need to record it, but with state removed. + hiddens_++; + (*photos_)[photo_id] = HIDE_PENDING; + } + + recompute_timestamp_range_ = true; +} + +void EpisodeTable_Episode::QuarantinePhoto(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + PhotoState* s = FindPtrOrNull(photos_.get(), photo_id); + if (s) { + if (*s == HIDDEN) { + quarantines_++; + hiddens_--; + *s = QUARANTINE_PENDING; + } else if (*s == POSTED) { + quarantines_++; + additions_--; + *s = QUARANTINE_PENDING; + } else if (*s == REMOVED) { + quarantines_++; + removals_--; + *s = QUARANTINE_PENDING; + } else { + CHECK_NE(*s, HIDE_PENDING) << "hidden pending"; + CHECK_NE(*s, POST_PENDING) << "post pending"; + CHECK_NE(*s, REMOVE_PENDING) << "remove pending"; + CHECK_NE(*s, UNSHARE_PENDING) << "unshare pending"; + } + } else { + // We've queried a photo as part of this episode which has been + // removed. We still need to record it, but with state unshared. + quarantines_++; + (*photos_)[photo_id] = QUARANTINE_PENDING; + } + + recompute_timestamp_range_ = true; +} + +void EpisodeTable_Episode::RemovePhoto(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + PhotoState* s = FindPtrOrNull(photos_.get(), photo_id); + if (s) { + if (*s == HIDDEN) { + hiddens_--; + removals_++; + *s = REMOVE_PENDING; + } else if (*s == POSTED) { + additions_--; + removals_++; + *s = REMOVE_PENDING; + } else if (*s == QUARANTINED) { + quarantines_--; + removals_++; + *s = REMOVE_PENDING; + } else { + CHECK_NE(*s, HIDE_PENDING) << "hidden pending"; + CHECK_NE(*s, POST_PENDING) << "post pending"; + CHECK_NE(*s, QUARANTINE_PENDING) << "quarantine pending"; + CHECK_NE(*s, UNSHARE_PENDING) << "unshare pending"; + } + } else { + // We've queried a photo as part of this episode which has been + // removed. We still need to record it, but with state removed. + removals_++; + (*photos_)[photo_id] = REMOVE_PENDING; + } + + recompute_timestamp_range_ = true; +} + +void EpisodeTable_Episode::UnsharePhoto(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + PhotoState* s = FindPtrOrNull(photos_.get(), photo_id); + if (s) { + if (*s == HIDDEN) { + hiddens_--; + unshares_++; + *s = UNSHARE_PENDING; + } else if (*s == POSTED) { + additions_--; + unshares_++; + *s = UNSHARE_PENDING; + } else if (*s == QUARANTINED) { + quarantines_--; + unshares_++; + *s = UNSHARE_PENDING; + } else if (*s == REMOVED) { + removals_--; + unshares_++; + *s = UNSHARE_PENDING; + } else { + CHECK_NE(*s, HIDE_PENDING) << "hidden pending"; + CHECK_NE(*s, POST_PENDING) << "post pending"; + CHECK_NE(*s, QUARANTINE_PENDING) << "quarantine pending"; + CHECK_NE(*s, REMOVE_PENDING) << "remove pending"; + } + } else { + // We've queried a photo as part of this episode which has been + // removed. We still need to record it, but with state unshared. + unshares_++; + (*photos_)[photo_id] = UNSHARE_PENDING; + } + + recompute_timestamp_range_ = true; +} + +bool EpisodeTable_Episode::IsHidden(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + const PhotoState state = FindOrDefault(*photos_, photo_id, REMOVED); + return (state == HIDDEN || state == HIDE_PENDING); +} + +bool EpisodeTable_Episode::IsPosted(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + const PhotoState state = FindOrDefault(*photos_, photo_id, REMOVED); + return (state == POSTED || state == POST_PENDING); +} + +bool EpisodeTable_Episode::IsQuarantined(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + const PhotoState state = FindOrDefault(*photos_, photo_id, REMOVED); + return (state == QUARANTINED || state == QUARANTINE_PENDING); +} + +bool EpisodeTable_Episode::IsRemoved(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + const PhotoState state = FindOrDefault(*photos_, photo_id, REMOVED); + return (state == REMOVED || state == REMOVE_PENDING); +} + +bool EpisodeTable_Episode::IsUnshared(int64_t photo_id) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + const PhotoState state = FindOrDefault(*photos_, photo_id, REMOVED); + return (state == UNSHARED || state == UNSHARE_PENDING); +} + +int EpisodeTable_Episode::CountPhotos() { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + int count = 0; + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + if (iter->second == POSTED || iter->second == POST_PENDING) { + ++count; + } + } + return count; +} + +void EpisodeTable_Episode::ListPhotos(vector* photo_ids) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + const bool posted = (iter->second == POSTED || iter->second == POST_PENDING); + if (posted) { + photo_ids->push_back(iter->first); + } + } +} + +void EpisodeTable_Episode::ListAllPhotos(vector* photo_ids) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + photo_ids->push_back(iter->first); + } +} + +void EpisodeTable_Episode::ListUnshared(vector* unshared_ids) { + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + if (iter->second == UNSHARED || iter->second == UNSHARE_PENDING) { + unshared_ids->push_back(iter->first); + } + } +} + +bool EpisodeTable_Episode::InLibrary() { + if (!has_viewpoint_id()) { + // If there's no viewpoint, user hasn't uploaded episode; always show. + return true; + } + // Otherwise, show any episode which is part of the default viewpoint. + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(viewpoint_id(), db_); + return vh.get() && vh->is_default(); +} + +bool EpisodeTable_Episode::GetTimeRange( + WallTime* earliest, WallTime* latest) { + if (!has_earliest_photo_timestamp() && !has_latest_photo_timestamp()) { + return false; + } + *earliest = earliest_photo_timestamp(); + *latest = latest_photo_timestamp(); + return true; +} + +bool EpisodeTable_Episode::GetLocation(Location* loc, Placemark* pm) { + if (resolved_location_) { + if (!location_.get()) { + return false; + } + *loc = *location_; + if (pm) { + *pm = *placemark_; + } + return true; + } + resolved_location_ = true; + location_.reset(new Location); + placemark_.reset(new Placemark); + + vector photo_ids; + ListPhotos(&photo_ids); + for (int i = 0; i < photo_ids.size(); ++i) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_ids[i], db_); + if (ph.get() && + ph->GetLocation(location_.get(), placemark_.get())) { + *loc = *location_; + if (pm) { + *pm = *placemark_; + } + return true; + } + } + location_.reset(NULL); + placemark_.reset(NULL); + return false; +} + +bool EpisodeTable_Episode::MaybeSetServerId() { + const int64_t device_id = state_->device_id(); + if (id().has_server_id() || !device_id) { + return false; + } + mutable_id()->set_server_id( + EncodeEpisodeId(device_id, id().local_id(), timestamp())); + return true; +} + +string EpisodeTable_Episode::FormatLocation(bool shorten) { + Location loc; + Placemark pm; + + if (GetLocation(&loc, &pm)) { + string loc_str; + state_->placemark_histogram()->FormatLocation(loc, pm, shorten, &loc_str); + return ToUppercase(loc_str); + } + return shorten ? "" : "Location Unavailable"; +} + +string EpisodeTable_Episode::FormatTimeRange(bool shorten, WallTime now) { + WallTime earliest, latest; + if (!GetTimeRange(&earliest, &latest)) { + return ""; + } + + // If shorten is true, format just the earliest time. + if (shorten) { + return FormatRelativeTime(earliest, now == 0 ? earliest : now); + } + // Format a time range, using start of the day corresponding + // to the latest time as "now". This results in the time + // range being expressed as times only. + return FormatDateRange(earliest, latest, now == 0 ? latest : now); +} + +string EpisodeTable_Episode::FormatContributor(bool shorten) { + if (!has_user_id() || user_id() == state_->user_id()) { + return ""; + } + if (shorten) { + return state_->contact_manager()->FirstName(user_id()); + } else { + return state_->contact_manager()->FullName(user_id()); + } +} + +void EpisodeTable_Episode::Invalidate(const DBHandle& updates) { + typedef ContentTable::Content Content; + EpisodeHandle eh(reinterpret_cast(this)); + state_->day_table()->InvalidateEpisode(eh, updates); + + // Invalidate any activities which shared photos from this episode. + if (id().has_server_id()) { + vector activity_ids; + state_->activity_table()->ListEpisodeActivities(id().server_id(), &activity_ids, updates); + for (int i = 0; i < activity_ids.size(); ++i) { + ActivityHandle ah = state_->activity_table()->LoadActivity(activity_ids[i], updates); + state_->day_table()->InvalidateActivity(ah, updates); + } + } +} + +bool EpisodeTable_Episode::Load() { + disk_timestamp_ = timestamp(); + return true; +} + +void EpisodeTable_Episode::SaveHook(const DBHandle& updates) { + bool has_posted_photo = false; + bool update_episode_timestamp = false; + if (photos_.get()) { + if (recompute_timestamp_range_) { + clear_earliest_photo_timestamp(); + clear_latest_photo_timestamp(); + } + + // Persist any photo additions/removals. + int photo_count = 0; + int updated = 0; + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + const int64_t photo_id = iter->first; + const string episode_photo_key = EncodeEpisodePhotoKey(id().local_id(), photo_id); + const string photo_episode_key = EncodePhotoEpisodeKey(photo_id, id().local_id()); + + // Keep earliest and latest photo timestamps up to date. Note that in the + // common case where a photo is being added, we're simply updating the + // existing range and not recomputing it from scratch. + if ((recompute_timestamp_range_ && iter->second == POSTED) || + iter->second == POST_PENDING) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_id, updates); + if (!ph.get()) { + LOG("couldn't load photo %d", photo_id); + continue; + } + // Keep earliest and latest photo timestamps up-to-date. + if (!has_earliest_photo_timestamp() || + ph->timestamp() < earliest_photo_timestamp()) { + set_earliest_photo_timestamp(ph->timestamp()); + } + if (!has_latest_photo_timestamp() || + ph->timestamp() > latest_photo_timestamp()) { + set_latest_photo_timestamp(ph->timestamp()); + } + } + + // Counts of the number of episodes the photo resides in immediately + // before/after performing an addition or deletion. Used to determine if + // the total count of photos in episodes is changing. + int pre_add_episode_count = -1; + int post_delete_episode_count = -1; + + if (iter->second == HIDE_PENDING) { + pre_add_episode_count = state_->episode_table()->CountEpisodes(photo_id, updates); + updates->Put(episode_photo_key, EpisodeTable::kHiddenValue); + updates->Put(photo_episode_key, string()); + iter->second = HIDDEN; + ++updated; + } else if (iter->second == POST_PENDING) { + pre_add_episode_count = state_->episode_table()->CountEpisodes(photo_id, updates); + updates->Put(episode_photo_key, EpisodeTable::kPostedValue); + updates->Put(photo_episode_key, string()); + iter->second = POSTED; + ++updated; + } else if (iter->second == QUARANTINE_PENDING) { + // NOTE: quarantined photos count against total count of photos. + pre_add_episode_count = state_->episode_table()->CountEpisodes(photo_id, updates); + updates->Put(episode_photo_key, EpisodeTable::kQuarantinedValue); + updates->Put(photo_episode_key, string()); + iter->second = QUARANTINED; + ++updated; + } else if (iter->second == REMOVE_PENDING) { + updates->Put(episode_photo_key, EpisodeTable::kRemovedValue); + if (updates->Exists(photo_episode_key)) { + // Decrement the photo ref count for this episode. + updates->Delete(photo_episode_key); + } + post_delete_episode_count = state_->episode_table()->CountEpisodes(photo_id, updates); + iter->second = REMOVED; + ++updated; + } else if (iter->second == UNSHARE_PENDING) { + updates->Put(episode_photo_key, EpisodeTable::kUnsharedValue); + if (updates->Exists(photo_episode_key)) { + updates->Delete(photo_episode_key); + post_delete_episode_count = state_->episode_table()->CountEpisodes(photo_id, updates); + } + iter->second = UNSHARED; + ++updated; + } + if (iter->second == POSTED) { + has_posted_photo = true; + } + if (pre_add_episode_count == 0) { + // Photo was added to its first episode. + ++photo_count; + } + if (post_delete_episode_count == 0) { + // Photo was deleted from its last episode. Delete any images + // associated with the photo. + state_->photo_table()->DeleteAllImages(photo_id, updates); + --photo_count; + } + } + + if (photo_count != 0) { + VLOG("saved episode had net addition of %d photos", photo_count); + } + + if (!has_earliest_photo_timestamp()) { + DCHECK(!has_latest_photo_timestamp()); + set_earliest_photo_timestamp(timestamp()); + set_latest_photo_timestamp(timestamp()); + } + + // Update the episode timestamp mapping any time a photo is added, + // hidden, removed or unshared. + update_episode_timestamp = updated > 0; + } + + if (disk_timestamp_ != timestamp() || update_episode_timestamp) { + if (disk_timestamp_ > 0) { + updates->Delete( + EncodeEpisodeTimestampKey(disk_timestamp_, id().local_id())); + } + disk_timestamp_ = timestamp(); + } + // We reference has_posted_photo to avoid counting photos in the episode + // unnecessarily. + if (disk_timestamp_ > 0 && (has_posted_photo || CountPhotos() > 0)) { + // Only add the episode to the , map if the + // episode contains POSTED photos. + updates->Put(EncodeEpisodeTimestampKey(disk_timestamp_, id().local_id()), + string()); + } + if (has_parent_id()) { + updates->Put(EncodeEpisodeParentChildKey(parent_id().local_id(), id().local_id()), + string()); + } + + additions_ = 0; + hiddens_ = 0; + quarantines_ = 0; + removals_ = 0; + unshares_ = 0; + recompute_timestamp_range_ = false; + + Invalidate(updates); +} + +void EpisodeTable_Episode::DeleteHook(const DBHandle& updates) { + int photo_count = 0; + MutexLock lock(&photos_mu_); + EnsurePhotoState(); + for (PhotoStateMap::iterator iter(photos_->begin()); + iter != photos_->end(); + ++iter) { + const int64_t photo_id = iter->first; + const string episode_photo_key = EncodeEpisodePhotoKey(id().local_id(), photo_id); + const string photo_episode_key = EncodePhotoEpisodeKey(photo_id, id().local_id()); + + updates->Delete(episode_photo_key); + updates->Delete(photo_episode_key); + + if (!state_->episode_table()->CountEpisodes(photo_id, updates)) { + // Photo was deleted from its last episode. Delete any images + // associated with the photo. + state_->photo_table()->DeleteAllImages(photo_id, updates); + --photo_count; + } + } + + if (photo_count != 0) { + VLOG("deleted episode had net addition of %d photos", photo_count); + } + + if (disk_timestamp_ > 0) { + updates->Delete( + EncodeEpisodeTimestampKey(disk_timestamp_, id().local_id())); + } + if (has_parent_id()) { + updates->Delete(EncodeEpisodeParentChildKey(parent_id().local_id(), id().local_id())); + } + // Delete episode selection key. + if (id().has_server_id()) { + updates->Delete(DBFormat::episode_selection_key(id().server_id())); + } + + Invalidate(updates); +} + +void EpisodeTable_Episode::EnsurePhotoState() { + photos_mu_.AssertHeld(); + if (have_photo_state_) { + return; + } + if (!photos_.get()) { + photos_.reset(new PhotoStateMap); + } + have_photo_state_ = true; + + for (ScopedPtr iter( + new EpisodeTable::EpisodePhotoIterator(id().local_id(), db_)); + !iter->done(); + iter->Next()) { + const int64_t photo_id = iter->photo_id(); + if (ContainsKey(*photos_, photo_id)) { + // Note that the disk photo state gets layered in underneath the existing + // photo state. + continue; + } + const Slice value = iter->value(); + if (value == EpisodeTable::kHiddenValue) { + (*photos_)[photo_id] = HIDDEN; + } else if (value == EpisodeTable::kPostedValue) { + (*photos_)[photo_id] = POSTED; + } else if (value == EpisodeTable::kQuarantinedValue) { + (*photos_)[photo_id] = QUARANTINED; + } else if (value == EpisodeTable::kRemovedValue) { + (*photos_)[photo_id] = REMOVED; + } else if (value == EpisodeTable::kUnsharedValue) { + (*photos_)[photo_id] = UNSHARED; + } + } +} + + +//// +// EpisodeIterator + +EpisodeTable::EpisodeIterator::EpisodeIterator( + EpisodeTable* table, WallTime start, bool reverse, const DBHandle& db) + : ContentIterator(db->NewIterator(), reverse), + table_(table), + db_(db), + timestamp_(0), + episode_id_(0) { + Seek(start); +} + +EpisodeHandle EpisodeTable::EpisodeIterator::GetEpisode() { + if (done()) { + return EpisodeHandle(); + } + return table_->LoadContent(episode_id_, db_); +} + +void EpisodeTable::EpisodeIterator::Seek(WallTime seek_time) { + ContentIterator::Seek(EncodeEpisodeTimestampKey( + seek_time, reverse_ ? std::numeric_limits::max() : 0)); +} + +bool EpisodeTable::EpisodeIterator::IteratorDone(const Slice& key) { + return !key.starts_with(kEpisodeTimestampKeyPrefix); +} + +bool EpisodeTable::EpisodeIterator::UpdateStateHook(const Slice& key) { + return DecodeEpisodeTimestampKey(key, ×tamp_, &episode_id_); +} + + +//// +// EpisodePhotoIterator + +EpisodeTable::EpisodePhotoIterator::EpisodePhotoIterator( + int64_t episode_id, const DBHandle& db) + : ContentIterator(db->NewIterator(), false), + episode_prefix_(EncodeEpisodePhotoKey(episode_id, 0)), + photo_id_(0) { + Seek(episode_prefix_); +} + +bool EpisodeTable::EpisodePhotoIterator::IteratorDone(const Slice& key) { + return !key.starts_with(episode_prefix_); +} + +bool EpisodeTable::EpisodePhotoIterator::UpdateStateHook(const Slice& key) { + int64_t episode_id; + return DecodeEpisodePhotoKey(key, &episode_id, &photo_id_); +} + + +//// +// EpisodeTable + +const string EpisodeTable::kHiddenValue = "h"; +const string EpisodeTable::kPostedValue = "a"; +const string EpisodeTable::kQuarantinedValue = "q"; +const string EpisodeTable::kRemovedValue = "r"; +const string EpisodeTable::kUnsharedValue = "u"; + + +EpisodeTable::EpisodeTable(AppState* state) + : ContentTable(state, + DBFormat::episode_key(), + DBFormat::episode_server_key(), + kEpisodeFSCKVersion, + DBFormat::metadata_key("episode_table_fsck")), + stats_initialized_(false), + episode_index_(new FullTextIndex(state_, kEpisodeIndexName)), + location_index_(new FullTextIndex(state_, kLocationIndexName)) { +} + +EpisodeTable::~EpisodeTable() { +} + +void EpisodeTable::Reset() { + MutexLock l(&stats_mu_); + stats_initialized_ = false; + stats_.Clear(); +} + +EpisodeHandle EpisodeTable::LoadEpisode(const EpisodeId& id, const DBHandle& db) { + EpisodeHandle eh; + if (id.has_local_id()) { + eh = LoadEpisode(id.local_id(), db); + } + if (!eh.get() && id.has_server_id()) { + eh = LoadEpisode(id.server_id(), db); + } + return eh; +} + +EpisodeTable::ContentHandle EpisodeTable::MatchPhotoToEpisode( + const PhotoHandle& p, const DBHandle& db) { + const WallTime max_time = p->timestamp() + kMaxTimeDist; + const WallTime min_time = std::max(0, p->timestamp() - kMaxTimeDist); + for (ScopedPtr iter( + NewEpisodeIterator(min_time, false, db)); + !iter->done() && iter->timestamp() <= max_time; + iter->Next()) { + EpisodeHandle e = LoadEpisode(iter->episode_id(), db); + if (!e.get() || + // Only match to episodes owned by this user. + e->GetUserId() != p->GetUserId() || + // Server disallows match to an episode created on another device. + e->GetDeviceId() != p->GetDeviceId() || + // Don't match a photo to an episode which is a reshare! + e->has_parent_id()) { + continue; + } + + // We use a photo iterator instead of ListPhotos() because we most likely + // will match on the first photo. + for (ScopedPtr photo_iter( + new EpisodeTable::EpisodePhotoIterator(e->id().local_id(), db)); + !photo_iter->done(); + photo_iter->Next()) { + if (photo_iter->value() != EpisodeTable::kPostedValue) { + continue; + } + PhotoHandle q = state_->photo_table()->LoadPhoto(photo_iter->photo_id(), db); + if (!q.get()) { + continue; + } + if (p->has_timestamp() && q->has_timestamp()) { + const double time_dist = fabs(p->timestamp() - q->timestamp()); + if (time_dist >= kMaxTimeDist) { + continue; + } + } + if (p->has_location() && q->has_location()) { + const double loc_dist = DistanceBetweenLocations( + p->location(), q->location()); + if (loc_dist >= kMaxLocDist) { + continue; + } + } + return e; + } + } + return EpisodeHandle(); +} + +void EpisodeTable::AddPhotoToEpisode(const PhotoHandle& p, const DBHandle& updates) { + if (!p->ShouldAddPhotoToEpisode()) { + return; + } + EpisodeHandle e = MatchPhotoToEpisode(p, updates); + if (!e.get()) { + e = NewEpisode(updates); + e->Lock(); + e->set_timestamp(p->timestamp()); + e->set_upload_episode(true); + e->MaybeSetServerId(); + VLOG("photo: new episode: %s", e->id()); + } else { + e->Lock(); + } + p->mutable_episode_id()->CopyFrom(e->id()); + e->AddPhoto(p->id().local_id()); + e->SaveAndUnlock(updates); +} + +int EpisodeTable::CountEpisodes(int64_t photo_id, const DBHandle& db) { + int count = 0; + for (DB::PrefixIterator iter(db, EncodePhotoEpisodeKey(photo_id, 0)); + iter.Valid(); + iter.Next()) { + int64_t photo_id; + int64_t episode_id; + if (DecodePhotoEpisodeKey(iter.key(), &photo_id, &episode_id)) { + ++count; + } + } + return count; +} + +bool EpisodeTable::ListEpisodes( + int64_t photo_id, vector* episode_ids, const DBHandle& db) { + for (DB::PrefixIterator iter(db, EncodePhotoEpisodeKey(photo_id, 0)); + iter.Valid(); + iter.Next()) { + int64_t photo_id; + int64_t episode_id; + if (DecodePhotoEpisodeKey(iter.key(), &photo_id, &episode_id)) { + if (episode_ids) { + episode_ids->push_back(episode_id); + } else { + return true; + } + } + } + return episode_ids && !episode_ids->empty(); +} + +bool EpisodeTable::ListLibraryEpisodes( + int64_t photo_id, vector* episode_ids, const DBHandle& db) { + vector raw_episode_ids; + if (!ListEpisodes(photo_id, &raw_episode_ids, db)) { + return false; + } + for (int i = 0; i < raw_episode_ids.size(); ++i) { + EpisodeHandle eh = LoadEpisode(raw_episode_ids[i], db); + if (eh.get() && eh->InLibrary()) { + if (episode_ids) { + episode_ids->push_back(raw_episode_ids[i]); + } else { + return true; + } + } + } + return episode_ids && !episode_ids->empty(); +} + +void EpisodeTable::RemovePhotos( + const PhotoSelectionVec& photo_ids, const DBHandle& updates) { + typedef std::unordered_map > EpisodeToPhotoMap; + EpisodeToPhotoMap episodes; + for (int i = 0; i < photo_ids.size(); ++i) { + episodes[photo_ids[i].episode_id].push_back(photo_ids[i].photo_id); + } + + ServerOperation op; + ServerOperation::RemovePhotos* r = op.mutable_remove_photos(); + + // Process the episodes in the same order as specified in the photo_ids + // vector to ease testing. + for (int i = 0; !episodes.empty() && i < photo_ids.size(); ++i) { + const int64_t episode_id = photo_ids[i].episode_id; + const vector* v = FindPtrOrNull(episodes, episode_id); + if (!v) { + continue; + } + const EpisodeHandle eh = LoadEpisode(episode_id, updates); + if (!eh.get()) { + episodes.erase(episode_id); + continue; + } + eh->Lock(); + + ActivityMetadata::Episode* e = NULL; + for (int j = 0; j < v->size(); ++j) { + const int64_t photo_id = (*v)[j]; + if (!state_->photo_table()->LoadPhoto(photo_id, updates).get()) { + continue; + } + if (!e) { + // Only add an episode to the server operation when the first valid + // photo id is found. + e = r->add_episodes(); + e->mutable_episode_id()->CopyFrom(eh->id()); + } + e->add_photo_ids()->set_local_id(photo_id); + eh->RemovePhoto(photo_id); + } + + if (e) { + eh->SaveAndUnlock(updates); + } else { + eh->Unlock(); + } + + episodes.erase(episode_id); + } + + if (r->episodes_size() > 0) { + // Only queue the operation if photos were removed. + op.mutable_headers()->set_op_id(state_->NewLocalOperationId()); + op.mutable_headers()->set_op_timestamp(WallTime_Now()); + state_->net_queue()->Add(PRIORITY_UI_ACTIVITY, op, updates); + } +} + +EpisodeHandle EpisodeTable::GetEpisodeForPhoto( + const PhotoHandle& p, const DBHandle& db) { + // Start with the photo's putative episode. + EpisodeHandle eh = LoadEpisode(p->episode_id(), db); + if (eh.get()) { + return eh; + } + + // Otherwise, get a list of all episodes the photo belongs to + // and find the first which has been uploaded, and for which the + // photo has neither been removed or unshared. + vector episode_ids; + ListEpisodes(p->id().local_id(), &episode_ids, db); + for (int i = 0; i < episode_ids.size(); ++i) { + eh = LoadEpisode(episode_ids[i], db); + if (eh.get() && + !eh->upload_episode() && + !eh->IsRemoved(p->id().local_id()) && + !eh->IsUnshared(p->id().local_id())) { + return eh; + } + } + + return EpisodeHandle(); +} + +void EpisodeTable::Validate( + const EpisodeSelection& s, const DBHandle& updates) { + const string key(DBFormat::episode_selection_key(s.episode_id())); + + // Load any existing episode selection and clear attributes which have been + // queried by "s". If no attributes remain set, the selection is deleted. + EpisodeSelection existing; + if (updates->GetProto(key, &existing)) { + if (s.get_attributes()) { + existing.clear_get_attributes(); + } + if (s.get_photos()) { + if (!existing.get_photos() || + s.photo_start_key() <= existing.photo_start_key()) { + existing.clear_get_photos(); + existing.clear_photo_start_key(); + } + } else if (existing.get_photos()) { + existing.set_photo_start_key( + std::max(existing.photo_start_key(), + s.photo_start_key())); + } + } + + if (existing.has_get_attributes() || + existing.has_get_photos()) { + updates->PutProto(key, existing); + } else { + updates->Delete(key); + } +} + +void EpisodeTable::Invalidate( + const EpisodeSelection& s, const DBHandle& updates) { + const string key(DBFormat::episode_selection_key(s.episode_id())); + + // Load any existing episode selection and merge invalidations from "s". + EpisodeSelection existing; + if (!updates->GetProto(key, &existing)) { + existing.set_episode_id(s.episode_id()); + } + + if (s.get_attributes()) { + existing.set_get_attributes(true); + } + if (s.get_photos()) { + if (existing.get_photos()) { + existing.set_photo_start_key(std::min(existing.photo_start_key(), + s.photo_start_key())); + } else { + existing.set_photo_start_key(s.photo_start_key()); + } + existing.set_get_photos(true); + } + + updates->PutProto(key, existing); +} + +void EpisodeTable::ListInvalidations( + vector* v, int limit, const DBHandle& db) { + v->clear(); + ScopedPtr iter(db->NewIterator()); + iter->Seek(kEpisodeSelectionKeyPrefix); + while (iter->Valid() && (limit <= 0 || v->size() < limit)) { + Slice key = ToSlice(iter->key()); + if (!key.starts_with(kEpisodeSelectionKeyPrefix)) { + break; + } + EpisodeSelection eps; + if (db->GetProto(key, &eps)) { + v->push_back(eps); + } else { + LOG("unable to read episode selection at key %s", key); + } + iter->Next(); + } +} + +void EpisodeTable::ClearAllInvalidations(const DBHandle& updates) { + ScopedPtr iter(updates->NewIterator()); + iter->Seek(kEpisodeSelectionKeyPrefix); + for (; iter->Valid(); iter->Next()) { + Slice key = ToSlice(iter->key()); + if (!key.starts_with(kEpisodeSelectionKeyPrefix)) { + break; + } + updates->Delete(key); + } +} + +void EpisodeTable::ListEpisodesByParentId( + int64_t parent_id, vector* children, const DBHandle& db) { + for (DB::PrefixIterator iter(db, EncodeEpisodeParentChildKey(parent_id, 0)); + iter.Valid(); + iter.Next()) { + int64_t parent_id; + int64_t child_id; + if (DecodeEpisodeParentChildKey(iter.key(), &parent_id, &child_id)) { + children->push_back(child_id); + } + } +} + +EpisodeTable::EpisodeIterator* EpisodeTable::NewEpisodeIterator( + WallTime start, bool reverse, const DBHandle& db) { + return new EpisodeIterator(this, start, reverse, db); +} + +EpisodeStats EpisodeTable::stats() { + EnsureStatsInit(); + MutexLock l(&stats_mu_); + return stats_; +} + +bool EpisodeTable::FSCKImpl(int prev_fsck_version, const DBHandle& updates) { + LOG("FSCK: EpisodeTable"); + bool changes = false; + if (FSCKEpisode(prev_fsck_version, updates)) { + changes = true; + } + // Handle any duplicates in secondary indexes by timestamp. These can exist + // as a result of a server bug which rounded up timestamps. + if (FSCKEpisodeTimestampIndex(updates)) { + changes = true; + } + return changes; +} + +bool EpisodeTable::FSCKEpisode(int prev_fsck_version, const DBHandle& updates) { + bool changes = false; + for (DB::PrefixIterator iter(updates, DBFormat::episode_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + EpisodeMetadata em; + if (em.ParseFromArray(value.data(), value.size())) { + EpisodeHandle eh = LoadEpisode(em.id().local_id(), updates); + eh->Lock(); + bool save_eh = false; + if (key != EncodeContentKey(DBFormat::episode_key(), em.id().local_id())) { + LOG("FSCK: episode id %d does not equal key %s; deleting key and re-saving", + em.id().local_id(), key); + updates->Delete(key); + save_eh = true; + } + + // Check required fields. + if (!eh->has_id() || !eh->has_timestamp()) { + LOG("FSCK: episode missing required fields: %s", *eh); + } + + // Check viewpoint; lookup first by server id. + if (eh->has_viewpoint_id()) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(eh->viewpoint_id(), updates); + if (vh.get() && !eh->viewpoint_id().local_id()) { + LOG("FSCK: missing local id for viewpoint %s", vh->id()); + eh->mutable_viewpoint_id()->CopyFrom(vh->id()); + save_eh = true; + } else if (!vh.get()) { + if (eh->viewpoint_id().has_server_id()) { + LOG("FSCK: missing viewpoint %s; setting invalidation", eh->viewpoint_id()); + state_->viewpoint_table()->InvalidateFull(eh->viewpoint_id().server_id(), updates); + changes = true; + } else { + LOG("FSCK: invalid reference to viewpoint %s; clearing", eh->viewpoint_id()); + eh->clear_viewpoint_id(); + save_eh = true; + } + } + } + + // Check secondary indexes. + if (eh->has_timestamp() && eh->CountPhotos() > 0) { + const string ts_episode_key = EncodeEpisodeTimestampKey( + eh->timestamp(), eh->id().local_id()); + if (!updates->Exists(ts_episode_key)) { + LOG("FSCK: missing timestamp episode key"); + save_eh = true; + } + } + + // Verify photo timestamp range is set. + if (!eh->has_earliest_photo_timestamp() || + !eh->has_latest_photo_timestamp()) { + LOG("FSCK: missing photo timestamp range; recomputing..."); + eh->recompute_timestamp_range_ = true; + save_eh = true; + } + + if (save_eh) { + LOG("FSCK: rewriting episode %s", *eh); + eh->SaveAndUnlock(updates); + changes = true; + } else { + eh->Unlock(); + } + } + } + + return changes; +} + +bool EpisodeTable::FSCKEpisodeTimestampIndex(const DBHandle& updates) { + // Map from episode id to secondary index key. + std::unordered_map* episode_ids( + new std::unordered_map); + bool changes = false; + + for (DB::PrefixIterator iter(updates, kEpisodeTimestampKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + WallTime timestamp; + int64_t episode_id; + if (!DecodeEpisodeTimestampKey(key, ×tamp, &episode_id)) { + LOG("FSCK: unreadable episode timestamp secondary index: %s", key); + updates->Delete(key); + changes = true; + } else { + if (ContainsKey(*episode_ids, episode_id)) { + LOG("FSCK: episode timestamp secondary index contains duplicate entries for %d; " + "deleting earlier instance (%s)", episode_id, (*episode_ids)[episode_id]); + updates->Delete((*episode_ids)[episode_id]); + changes = true; + } + (*episode_ids)[episode_id] = ToString(key); + } + } + + delete episode_ids; + return changes; +} + +void EpisodeTable::Search(const Slice& query, EpisodeSearchResults* results) { + QueryRewriter rewriter(state_); + ScopedPtr parsed_query(rewriter.ParseAndRewrite(query)); + FullTextQueryIteratorBuilder builder({episode_index_.get(), location_index_.get()}, state_->db()); + for (ScopedPtr iter(builder.BuildIterator(*parsed_query)); + iter->Valid(); + iter->Next()) { + results->push_back(FastParseInt64(iter->doc_id())); + } +} + +void EpisodeTable::SaveContentHook(Episode* episode, const DBHandle& updates) { + vector terms; + vector location_terms; + int pos = 0; + int location_pos = 0; + // Don't index anything/remove all indexed terms if all photos have been removed. + if (episode->CountPhotos() > 0) { + Location loc; + Placemark pm; + // Index the location at several granularities. This lets us have separate autocomplete + // entries for "Soho, NYC" and "New York, NY". + if (episode->GetLocation(&loc, &pm)) { + StringSet seen_location_terms; + auto IndexLocationTerm = [&](const string& term) { + if (term.empty() || ContainsKey(seen_location_terms, term)) { + return; + } + seen_location_terms.insert(term); + // Index each location term as both a bag of words in the main index and a single term in the + // location index (for better autocomplete). + pos = episode_index_->ParseIndexTerms(pos, term, &terms); + // TODO(ben): this raw term should go through the denormalization process so "i" can + // autocomplete to "ÃŽle-de-France". + location_terms.push_back(FullTextIndexTerm(ToLowercase(term), term, location_pos++)); + }; + + IndexLocationTerm(FormatPlacemarkWithReferencePlacemark(pm, NULL, false, PM_SUBLOCALITY, 2)); + IndexLocationTerm(FormatPlacemarkWithReferencePlacemark(pm, NULL, false, PM_LOCALITY, 2)); + IndexLocationTerm(FormatPlacemarkWithReferencePlacemark(pm, NULL, false, PM_STATE, 2)); + } + if (episode->has_timestamp()) { + // Index the month, date and year. + const string date = FormatDate("%B %e %Y", episode->timestamp()); + pos = episode_index_->ParseIndexTerms(pos, date, &terms); + } + if (episode->user_id() != 0 && episode->user_id() != state_->user_id()) { + pos = episode_index_->AddVerbatimToken(pos, ContactManager::FormatUserToken(episode->user_id()), &terms); + } + if (episode->viewpoint_id().local_id() != 0) { + pos = episode_index_->AddVerbatimToken(pos, ViewpointTable::FormatViewpointToken(episode->viewpoint_id().local_id()), &terms); + } + } + episode_index_->UpdateIndex(terms, ToString(episode->id().local_id()), + FullTextIndex::TimestampSortKey(episode->timestamp()), + episode->mutable_indexed_terms(), updates); + location_index_->UpdateIndex(location_terms, ToString(episode->id().local_id()), + FullTextIndex::TimestampSortKey(episode->timestamp()), + episode->mutable_indexed_location_terms(), updates); + + EnsureStatsInit(); + MutexLock l(&stats_mu_); + stats_.set_hidden_photos(stats_.hidden_photos() + episode->hiddens()); + stats_.set_posted_photos(stats_.posted_photos() + episode->additions()); + stats_.set_quarantined_photos(stats_.quarantined_photos() + episode->quarantines()); + stats_.set_removed_photos(stats_.removed_photos() + episode->removals()); + stats_.set_unshared_photos(stats_.unshared_photos() + episode->unshares()); +} + +void EpisodeTable::DeleteContentHook(Episode* episode, const DBHandle& updates) { + episode_index_->RemoveTerms(episode->mutable_indexed_terms(), updates); + location_index_->RemoveTerms(episode->mutable_indexed_location_terms(), updates); +} + +void EpisodeTable::EnsureStatsInit() { + if (stats_initialized_) { + return; + } + + MutexLock l(&stats_mu_); + stats_initialized_ = true; + // ScopedTimer timer("episode stats"); + + // The stats could not be loaded; Regenerate from scratch. + int hidden_photos = 0; + int posted_photos = 0; + int quarantined_photos = 0; + int removed_photos = 0; + int unshared_photos = 0; + for (DB::PrefixIterator iter(state_->db(), EncodeEpisodePhotoKey(0, 0)); + iter.Valid(); + iter.Next()) { + const Slice value = iter.value(); + int64_t episode_id; + int64_t photo_id; + if (DecodeEpisodePhotoKey(iter.key(), &episode_id, &photo_id)) { + if (value == EpisodeTable::kHiddenValue) { + hidden_photos++; + } else if (value == EpisodeTable::kPostedValue) { + posted_photos++; + } else if (value == EpisodeTable::kQuarantinedValue) { + quarantined_photos++; + } else if (value == EpisodeTable::kRemovedValue) { + removed_photos++; + } else if (value == EpisodeTable::kUnsharedValue) { + unshared_photos++; + } + } + } + + stats_.set_hidden_photos(hidden_photos); + stats_.set_posted_photos(posted_photos); + stats_.set_quarantined_photos(quarantined_photos); + stats_.set_removed_photos(removed_photos); + stats_.set_unshared_photos(unshared_photos); +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/EpisodeTable.h b/clients/shared/EpisodeTable.h new file mode 100644 index 0000000..4c86e98 --- /dev/null +++ b/clients/shared/EpisodeTable.h @@ -0,0 +1,373 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_EPISODE_TABLE_H +#define VIEWFINDER_EPISODE_TABLE_H + +#import "ContentTable.h" +#import "DB.h" +#import "EpisodeMetadata.pb.h" +#import "EpisodeStats.pb.h" +#import "InvalidateMetadata.pb.h" +#import "PhotoSelection.h" +#import "PhotoTable.h" + +class FullTextIndex; + +// The EpisodeTable class maintains the mappings: +// -> +// -> +// -> +// , -> <> (list children by parent id) +// , -> (post table) +// , -> <> (photo ref-count table) +// , -> <> (episode by date table) +// +// EpisodeTable is thread-safe and EpisodeHandle is thread-safe, but individual +// Episodes are not. + +class EpisodeTable_Episode : public EpisodeMetadata { + friend class EpisodeTable; + + enum PhotoState { + HIDE_PENDING = 0, + POST_PENDING, + QUARANTINE_PENDING, + REMOVE_PENDING, + UNSHARE_PENDING, + HIDDEN, + POSTED, + QUARANTINED, + REMOVED, + UNSHARED, + }; + typedef std::map PhotoStateMap; + + public: + virtual void MergeFrom(const EpisodeMetadata& m); + // Unimplemented; exists to get the compiler not to complain about hiding the base class's overloaded MergeFrom. + virtual void MergeFrom(const ::google::protobuf::Message&); + + // Return the device/user id for the episode. Returns + // AppState::{device,user}_id if no the episode does not have a device/user + // id set. + int64_t GetDeviceId() const; + int64_t GetUserId() const; + + // Adds a photo to the episode. The added photo is not persisted until + // Save() is called. + void AddPhoto(int64_t photo_id); + + // Hides a photo from the epsiode. Hiding photos does not affect + // accounting but removes the photo from the display. + void HidePhoto(int64_t photo_id); + + // Indicates that a photo in the episode has been quarantined. This + // does not remove the photo/episode ref count links, but will + // prevent further attempts to display the photo. The quarantined + // photo is not persisted until Save() is called. + void QuarantinePhoto(int64_t photo_id); + + // Removes a photo from the episode. This only removes the + // photo/episode ref count link for photos which are not shared. The + // removed photo is not persisted until Save() is called. This should + // only be done on photos which are part of the default viewpoint. + void RemovePhoto(int64_t photo_id); + + // Unshares a photo from the episode. This removes the photo/episode + // ref count link for shared photos (the call itself is only valid for + // shared photos). + void UnsharePhoto(int64_t photo_id); + + // Returns true if the photo is hidden from the episode. + bool IsHidden(int64_t photo_id); + + // Returns true if the photo is posted to the episode. + bool IsPosted(int64_t photo_id); + + // Returns true if the photo was quarantined from the episode. + bool IsQuarantined(int64_t photo_id); + + // Returns true if the photo was removed from the episode. + bool IsRemoved(int64_t photo_id); + + // Returns true if the photo was unshared from the episode. + bool IsUnshared(int64_t photo_id); + + // Returns a count of the number of photos posted to the episode. + int CountPhotos(); + + // Lists only the POSTED photos associated with the episode. + void ListPhotos(vector* photo_ids); + + // Lists all photos associated with the episode, regardless of + // state. This includes hidden, removed, unshared & + // quarantined. Use the IsHidden, IsPosted, IsRemoved, IsQuarantined + // & IsUnshared methods to check state of individual photos. + void ListAllPhotos(vector* photo_ids); + + // Lists the photos which have been unshared. + void ListUnshared(vector* unshared_ids); + + // Returns whether the episode is part of the user's photo library. + // This is true if the episode is unshared or not part of any viewpoint, + // or is part of the user's default viewpoint. + bool InLibrary(); + + // Returns time range from earliest to latest photo. If the episode + // contains no photos, returns false. + bool GetTimeRange(WallTime* earliest, WallTime* latest); + + // Returns true if any photo within this episode has a valid + // location. If "location" and/or "placemark" are non-NULL, sets + // their values if available. Since this method queries each photo + // individually to find one with a location and placemark, the value + // is cached after the first call and returned efficiently thereafter. + bool GetLocation(Location* location, Placemark* placemark); + + // Set the server id if it is not already set. Returns true iff the server-id + // was set. + bool MaybeSetServerId(); + + // Returns a formatted location. + string FormatLocation(bool shorten); + + // Returns a formatted time range from earliest to latest photo timestamps. + // Specifying "now" uses the specified date as the relative offset for + // formatting. If not specified, the time range is formatted relative to + // the latest timestamp of any photo in the episode. + string FormatTimeRange(bool shorten, WallTime now = 0); + + // Returns a formatted contributor. If the episode is owned by the + // user, returns empty string. Otherwise, returns full name if + // "shorten" is false or first name if "shorten" is true. + string FormatContributor(bool shorten); + + // Invalidates episode occurences in day metadata and all activities + // which shared photos from this episode. + void Invalidate(const DBHandle& updates); + + // Gets the pending count of photo additions, hiddens, quarantines, + // removals and unshares. + int additions() const { return additions_; } + int hiddens() const { return hiddens_; } + int quarantines() const { return quarantines_; } + int removals() const { return removals_; } + int unshares() const { return unshares_; } + + protected: + bool Load(); + void SaveHook(const DBHandle& updates); + void DeleteHook(const DBHandle& updates); + + int64_t local_id() const { return id().local_id(); } + const string& server_id() const { return id().server_id(); } + + EpisodeTable_Episode(AppState* state, const DBHandle& db, int64_t id); + + private: + void EnsurePhotoState(); + + protected: + AppState* state_; + DBHandle db_; + + private: + // The timestamp as stored on disk. + WallTime disk_timestamp_; + // A cache of the photos associated with the episode. Populated on-demand. + ScopedPtr photos_; + // Protects photos_. Should be held when calling EnsurePhotoState. + Mutex photos_mu_; + int additions_; + int hiddens_; + int quarantines_; + int removals_; + int unshares_; + bool have_photo_state_; + bool recompute_timestamp_range_; + // Cached values for GetLocation(). + bool resolved_location_; + ScopedPtr location_; + ScopedPtr placemark_; +}; + +class EpisodeTable : public ContentTable { + typedef EpisodeTable_Episode Episode; + + public: + class EpisodeIterator : public ContentIterator { + friend class EpisodeTable; + + public: + ContentHandle GetEpisode(); + + WallTime timestamp() const { return timestamp_; } + int64_t episode_id() const { return episode_id_; } + + // Position the iterator at the specified time. done() is true + // if there is no more content after seeking. + void Seek(WallTime seek_time); + + private: + EpisodeIterator( + EpisodeTable* table, WallTime start, bool reverse, const DBHandle& db); + + bool IteratorDone(const Slice& key); + bool UpdateStateHook(const Slice& key); + + private: + EpisodeTable* const table_; + DBHandle db_; + WallTime timestamp_; + int64_t episode_id_; + }; + + class EpisodePhotoIterator : public ContentIterator { + friend class EpisodeTable; + + public: + EpisodePhotoIterator(int64_t episode_id, const DBHandle& db); + + bool IteratorDone(const Slice& key); + bool UpdateStateHook(const Slice& key); + + int64_t photo_id() const { return photo_id_; } + + private: + const string episode_prefix_; + int64_t photo_id_; + }; + + public: + EpisodeTable(AppState* state); + ~EpisodeTable(); + + void Reset(); + + ContentHandle NewEpisode(const DBHandle& updates) { + return NewContent(updates); + } + ContentHandle LoadEpisode(int64_t id, const DBHandle& db) { + return LoadContent(id, db); + } + ContentHandle LoadEpisode(const string& server_id, const DBHandle& db) { + return LoadContent(server_id, db); + } + ContentHandle LoadEpisode(const EpisodeId& id, const DBHandle& db); + + // Find the episode to which the specified photo should be added. Might + // return NULL if a new episode should be created. + ContentHandle MatchPhotoToEpisode(const PhotoHandle& p, const DBHandle& db); + + // Add the photo to the best matching episode or create a new episode and add + // the photo. + void AddPhotoToEpisode(const PhotoHandle& p, const DBHandle& updates); + + // Returns a count of the number of episodes the specified photo is + // associated with. + int CountEpisodes(int64_t photo_id, const DBHandle& db); + + // Lists the episodes the specified photo is associated with. + // Returns whether any episodes were found and sets the list of + // matching episodes in *episode_ids. "episode_ids" maybe NULL. + bool ListEpisodes(int64_t photo_id, vector* episode_ids, const DBHandle& db); + + // Lists the episodes the specified photo is associated with which + // are part of the default viewpoint (e.g. visible in the personal + // library). Returns whether any such episodes were located. Sets + // *episode_ids with the list of episodes. "episode_ids" may be NULL. + bool ListLibraryEpisodes(int64_t photo_id, vector* episode_ids, const DBHandle& db); + + // Removes photos from episodes and queues up a server operation. The + // photo_ids vector consists of pairs. + void RemovePhotos(const PhotoSelectionVec& photo_ids, + const DBHandle& updates); + + // TODO(spencer): implement HidePhotos(). + // Hides photos from episodes and queues up a server operation. + + // Returns the most appropriate episode for the specified photo. We + // prefer the original episode (as listed in p->episode_id()) if + // available. Otherwise, try to locate a non-derived episode that the + // user has access to. + ContentHandle GetEpisodeForPhoto(const PhotoHandle& p, const DBHandle& db); + + // Validates portions of the specified episode. + void Validate(const EpisodeSelection& s, const DBHandle& updates); + + // Invalidates portions of the specified episode. + void Invalidate(const EpisodeSelection& s, const DBHandle& updates); + + // Lists the episode invalidations. + void ListInvalidations(vector* v, int limit, const DBHandle& db); + + // Clear all of the episode invalidations. + void ClearAllInvalidations(const DBHandle& updates); + + // Lists all episodes with matching parent id. + void ListEpisodesByParentId( + int64_t parent_id, vector* children, const DBHandle& db); + + // Returns a new EpisodeIterator object for iterating over the episodes in + // ascending timestamp order. The caller is responsible for deleting the + // iterator. + EpisodeIterator* NewEpisodeIterator(WallTime start, bool reverse, const DBHandle& db); + + // Returns aggregated stats of photos contained across all episodes. + EpisodeStats stats(); + + // Repairs secondary indexes and sanity checks episode metadata. + bool FSCKImpl(int prev_fsck_version, const DBHandle& updates); + + // Verifies and repairs references from episode metadata to other objects. + bool FSCKEpisode(int prev_fsck_version, const DBHandle& updates); + + // Verifies and repairs episode timestamp secondary index. + bool FSCKEpisodeTimestampIndex(const DBHandle& updates); + + // Returns local episode ids for episodes matching the query. + typedef vector EpisodeSearchResults; + void Search(const Slice& query, EpisodeSearchResults* results); + + FullTextIndex* episode_index() const { return episode_index_.get(); } + FullTextIndex* location_index() const { return location_index_.get(); } + + public: + static const string kHiddenValue; + static const string kPostedValue; + static const string kQuarantinedValue; + static const string kRemovedValue; + static const string kUnsharedValue; + + protected: + virtual void SaveContentHook(Episode* episode, const DBHandle& updates); + virtual void DeleteContentHook(Episode* episode, const DBHandle& updates); + + private: + void EnsureStatsInit(); + + private: + mutable Mutex stats_mu_; + bool stats_initialized_; + EpisodeStats stats_; + ScopedPtr episode_index_; + ScopedPtr location_index_; +}; + +typedef EpisodeTable::ContentHandle EpisodeHandle; + +string EncodeEpisodePhotoKey(int64_t episode_id, int64_t photo_id); +string EncodePhotoEpisodeKey(int64_t photo_id, int64_t episode_id); +string EncodeEpisodeTimestampKey(WallTime timestamp, int64_t episode_id); +string EncodeEpisodeParentChildKey(int64_t parent_id, int64_t child_id); +bool DecodeEpisodePhotoKey(Slice key, int64_t* episode_id, int64_t* photo_id); +bool DecodePhotoEpisodeKey(Slice key, int64_t* photo_id, int64_t* episode_id); +bool DecodeEpisodeTimestampKey(Slice key, WallTime* timestamp, int64_t* episode_id); +bool DecodeEpisodeParentChildKey(Slice key, int64_t* parent_id, int64_t* child_id); + +#endif // VIEWFINDER_EPISODE_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/FileUtils.android.cc b/clients/shared/FileUtils.android.cc new file mode 100644 index 0000000..bbfe2a4 --- /dev/null +++ b/clients/shared/FileUtils.android.cc @@ -0,0 +1,7 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault + +#include "FileUtils.h" + +// Android does not currently support cloud backup. +void FileExcludeFromBackup(const string& path, bool is_dir) { return; } diff --git a/clients/shared/FileUtils.cc b/clients/shared/FileUtils.cc new file mode 100644 index 0000000..0b179e1 --- /dev/null +++ b/clients/shared/FileUtils.cc @@ -0,0 +1,239 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#import "FileUtils.h" +#import "Logging.h" + +int FileCreate(const string& path, bool exclude_from_backup) { + const int new_fd = open(path.c_str(), O_CREAT|O_WRONLY, 0644); + if (new_fd < 0) { + VLOG("open failed: %s: %d (%s)", path, errno, strerror(errno)); + return -1; + } + if (exclude_from_backup) { + FileExcludeFromBackup(path); + } + return new_fd; +} + +bool FileExists(const string& path) { + struct stat s; + if (stat(path.c_str(), &s) < 0) { + return false; + } + return s.st_mode & S_IFREG; +} + +int64_t FileSize(const string& path) { + struct stat s; + if (stat(path.c_str(), &s) < 0) { + if (errno != ENOENT) { + VLOG("stat failed: %s: %d (%s)", path, errno, strerror(errno)); + } + return -1; + } + return s.st_size; +} + +int64_t FileSize(int fd) { + struct stat s; + if (fstat(fd, &s) < 0) { + VLOG("fstat failed: fd %d: %d (%s)", fd, errno, strerror(errno)); + return -1; + } + return s.st_size; +} + +bool FileRename(const string& old_path, const string& new_path) { + if (rename(old_path.c_str(), new_path.c_str()) < 0) { + VLOG("rename failed: %s -> %s: %d (%s)", + old_path, new_path, errno, strerror(errno)); + return false; + } + return true; +} + +bool FileRemove(const string& path) { + if (unlink(path.c_str()) < 0) { + if (errno != ENOENT) { + VLOG("remove failed: %s: %d (%s)", path, errno, strerror(errno)); + } + return false; + } + return true; +} + +bool DirCreate(const string& path, int mode, bool exclude_from_backup) { + if (mkdir(path.c_str(), mode) < 0) { + if (errno != EEXIST) { + VLOG("mkdir failed: %s: %d (%s)", path, errno, strerror(errno)); + return false; + } + return true; + } + if (exclude_from_backup) { + FileExcludeFromBackup(path, true); + } + return true; +} + +bool DirExists(const string& path) { + struct stat s; + if (stat(path.c_str(), &s) < 0) { + return false; + } + return s.st_mode & S_IFDIR; +} + +bool DirRemove(const string& path, bool recursive, int* files, int* dirs) { + if (recursive) { + DIR* dir = opendir(path.c_str()); + if (!dir) { + return false; + } + struct dirent* r = NULL; + while ((r = readdir(dir)) != 0) { + const string name(r->d_name); + if (name == "." || name == "..") { + continue; + } + const string subpath(path + "/" + name); + struct stat s; + if (lstat(subpath.c_str(), &s) < 0) { + continue; + } + if (s.st_mode & S_IFDIR) { + if (!DirRemove(subpath, true, files, dirs)) { + break; + } + } else { + if (!FileRemove(subpath)) { + break; + } + if (files) { + *files += 1; + } + } + } + closedir(dir); + } + + if (rmdir(path.c_str()) < 0) { + LOG("rmdir failed: %s: %d (%s)", path, errno, strerror(errno)); + return false; + } + if (dirs) { + *dirs += 1; + } + return true; +} + +bool DirList(const string& path, vector* files) { + DIR* dir = opendir(path.c_str()); + if (!dir) { + return false; + } + struct dirent* r = NULL; + while ((r = readdir(dir)) != 0) { + const string name(r->d_name); + if (name == "." || name == "..") { + continue; + } + files->push_back(name); + } + closedir(dir); + return true; +} + +bool WriteStringToFD(int fd, const Slice& str, bool silent) { + const char* p = str.data(); + int n = str.size(); + while (n > 0) { + ssize_t res = write(fd, p, n); + if (res < 0) { + if (!silent) { + LOG("write failed: %d (%s)", errno, strerror(errno)); + } + break; + } + p += res; + n -= res; + } + return (n == 0); +} + +bool WriteStringToFile(const string& path, const Slice& str, + bool exclude_from_backup) { + const string tmp_path(path + ".tmp"); + int fd = FileCreate(tmp_path, exclude_from_backup); + if (fd < 0) { + LOG("open failed: %s: %d (%s)", tmp_path, errno, strerror(errno)); + return false; + } + const bool res = WriteStringToFD(fd, str); + close(fd); + if (!res) { + FileRemove(tmp_path); + return false; + } + return FileRename(tmp_path, path); +} + +bool WriteProtoToFile( + const string& path, const google::protobuf::MessageLite& message, + bool exclude_from_backup) { + return WriteStringToFile( + path, message.SerializeAsString(), exclude_from_backup); +} + +bool ReadFileToString(const string& path, string* str) { + int fd = open(path.c_str(), O_RDONLY); + if (fd < 0) { + // LOG("open failed: %s: %d (%s)", path, errno, strerror(errno)); + return false; + } + struct stat s; + if (fstat(fd, &s) < 0) { + LOG("stat failed: %s: %d (%s)", path, errno, strerror(errno)); + return false; + } + + int n = s.st_size; + str->resize(n); + char* p = &(*str)[0]; + + while (n > 0) { + ssize_t res = read(fd, p, n); + if (res < 0) { + LOG("read failed: %s: %d (%s)", path, errno, strerror(errno)); + break; + } + p += res; + n -= res; + } + close(fd); + return n == 0; +} + +string ReadFileToString(const string& path) { + string s; + if (!ReadFileToString(path, &s)) { + return string(); + } + return s; +} + +bool ReadFileToProto( + const string& path, google::protobuf::MessageLite* message) { + string s; + if (!ReadFileToString(path, &s)) { + return false; + } + return message->ParseFromString(s); +} diff --git a/clients/shared/FileUtils.h b/clients/shared/FileUtils.h new file mode 100644 index 0000000..26bd2c2 --- /dev/null +++ b/clients/shared/FileUtils.h @@ -0,0 +1,48 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_FILE_UTILS_H +#define VIEWFINDER_FILE_UTILS_H + +#import "Utils.h" + +namespace google { +namespace protobuf { +class MessageLite; +} // namespace protobuf +} // namespace google + +int FileCreate(const string& path, bool exclude_from_backup = true); +bool FileExists(const string& path); +int64_t FileSize(const string& path); +int64_t FileSize(int fd); +bool FileRename(const string& old_path, const string& new_path); +bool FileRemove(const string& path); +void FileExcludeFromBackup(const string& path, bool is_dir = false); +// Creates the given directory. Returns true if the directory was created or +// already existed, otherwise VLOGs a warning and returns false. +bool DirCreate(const string& path, int mode = 0755, + bool exclude_from_backup = true); +bool DirExists(const string& path); +bool DirRemove(const string& path, bool recursive = false, + int* files = NULL, int* dirs = NULL); +bool DirList(const string& path, vector* files); +void DirExcludeFromBackup(const string& path, bool recursive = false); +bool WriteStringToFD(int fd, const Slice& str, bool silent = false); +bool WriteStringToFile(const string& path, const Slice& str, + bool exclude_from_backup = true); +bool WriteProtoToFile(const string& path, + const google::protobuf::MessageLite& message, + bool exclude_from_backup = true); +#ifdef __OBJC__ +bool WriteDataToFile(const string& path, NSData* data, + bool exclude_from_backup = true); +#endif // __OBJC__ +bool ReadFileToString(const string& path, string* str); +string ReadFileToString(const string& path); +bool ReadFileToProto(const string& path, google::protobuf::MessageLite* message); +#ifdef __OBJC__ +NSData* ReadFileToData(const string& path); +#endif // __OBJC__ + +#endif // VIEWFINDER_FILE_UTILS_H diff --git a/clients/shared/FileUtils.ios.mm b/clients/shared/FileUtils.ios.mm new file mode 100644 index 0000000..c48c633 --- /dev/null +++ b/clients/shared/FileUtils.ios.mm @@ -0,0 +1,97 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#import "FileUtils.h" +#import "Logging.h" + +void FileExcludeFromBackup(const string& path, bool is_dir) { + if (kIOSVersion == "5.0.1") { + const char* name = "com.apple.MobileBackup"; + uint8_t value = 1; + if (setxattr(path.c_str(), name, &value, sizeof(value), 0, 0) != 0) { + LOG("setxattr failed: %s: %d (%s)", path, errno, strerror(errno)); + } + } else if (kIOSVersion >= "5.1") { + NSURL* url = [NSURL fileURLWithPath:NewNSString(path) isDirectory:is_dir]; + NSError* error = NULL; + if (![url setResourceValue:[NSNumber numberWithBool: YES] + forKey:NSURLIsExcludedFromBackupKey + error:&error]) { + LOG("exclude from backup failed: %s: %s", path, error); + } + } +} + +void DirExcludeFromBackup(const string& path, bool recursive) { + if (recursive) { + DIR* dir = opendir(path.c_str()); + if (!dir) { + return; + } + struct dirent* r = NULL; + while ((r = readdir(dir)) != 0) { + const string name(r->d_name, r->d_namlen); + if (name == "." || name == "..") { + continue; + } + const string subpath(path + "/" + name); + struct stat s; + if (lstat(subpath.c_str(), &s) < 0) { + continue; + } + if (s.st_mode & S_IFDIR) { + DirExcludeFromBackup(subpath, true); + } else { + FileExcludeFromBackup(subpath); + } + } + closedir(dir); + } + + FileExcludeFromBackup(path, true); +} + +bool WriteDataToFile(const string& path, NSData* data, + bool exclude_from_backup) { + return WriteStringToFile( + path, Slice((const char*)data.bytes, data.length), + exclude_from_backup); +} + +NSData* ReadFileToData(const string& path) { + int fd = open(path.c_str(), O_RDONLY); + if (fd < 0) { + // LOG("open failed: %s: %d (%s)", path, errno, strerror(errno)); + return NULL; + } + struct stat s; + if (fstat(fd, &s) < 0) { + LOG("stat failed: %s: %d (%s)", path, errno, strerror(errno)); + return NULL; + } + + int n = s.st_size; + char* p = reinterpret_cast(malloc(n)); + NSData* data = [[NSData alloc] initWithBytesNoCopy:p + length:n + freeWhenDone:YES]; + + while (n > 0) { + ssize_t res = read(fd, p, n); + if (res < 0) { + LOG("read failed: %s: %d (%s)", path, errno, strerror(errno)); + data = NULL; + break; + } + p += res; + n -= res; + } + close(fd); + return data; +} diff --git a/clients/shared/FollowerGroup.proto b/clients/shared/FollowerGroup.proto new file mode 100644 index 0000000..c65f767 --- /dev/null +++ b/clients/shared/FollowerGroup.proto @@ -0,0 +1,25 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "FollowerGroupPB"; + +message FollowerGroup { + // List of user ids in the follower group. Sorted in ascending order. + repeated int64 user_ids = 1; + + // Information about each viewpoint followed by this group. + message ViewpointInfo { + optional int64 viewpoint_id = 1; + optional double earliest_timestamp = 2; + optional double latest_timestamp = 3; + optional double weight = 4; + } + + // List of viewpoints, sorted by viewpoint id. + repeated ViewpointInfo viewpoints = 2; + + // Group weight. Computed in-process and not persisted. + optional double weight = 3; +} + diff --git a/clients/shared/FontSymbols.cc b/clients/shared/FontSymbols.cc new file mode 100644 index 0000000..84a4758 --- /dev/null +++ b/clients/shared/FontSymbols.cc @@ -0,0 +1,15 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "FontSymbols.h" + +const string kSpaceSymbol = "\u2630"; +const string kTimeSymbol = "\u2631"; +const string kPhotoSymbol = "\u2632"; +const string kUserSymbol = "\u2633"; +const string kCommentSymbol = "\u2634"; +const string kConvoSymbol = "\u2635"; +const string kPhotomarkSymbol = "\u2636"; +const string kPhotomarkOffSymbol = "\u2637"; +const string kBoldSpaceSymbol = "\u2630"; +const string kBoldCompassSymbol = "\u2638"; diff --git a/clients/shared/FontSymbols.h b/clients/shared/FontSymbols.h new file mode 100644 index 0000000..d83a346 --- /dev/null +++ b/clients/shared/FontSymbols.h @@ -0,0 +1,20 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_FONT_SYMBOLS_H +#define VIEWFINDER_FONT_SYMBOLS_H + +#import "Utils.h" + +extern const string kSpaceSymbol; +extern const string kTimeSymbol; +extern const string kPhotoSymbol; +extern const string kUserSymbol; +extern const string kCommentSymbol; +extern const string kConvoSymbol; +extern const string kPhotomarkSymbol; +extern const string kPhotomarkOffSymbol; +extern const string kBoldSpaceSymbol; +extern const string kBoldCompassSymbol; + +#endif // VIEWFINDER_FONT_SYMBOLS_H diff --git a/clients/shared/Format.cc b/clients/shared/Format.cc new file mode 100644 index 0000000..e01b538 --- /dev/null +++ b/clients/shared/Format.cc @@ -0,0 +1,340 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// +// TODO(pmattis): %|spec| item specifier + +#import +#import "Format.h" +#import "Logging.h" + +namespace { + +using std::ios_base; +using std::min; + +struct State { + State(ostream& o, const string& format, + const Formatter::Arg* const* a, int count) + : os(o), + ptr(format.data()), + end(ptr + format.size()), + args(a), + args_count(count), + num_args(0), + cur_arg(0) { + } + + ostream& os; + const char* ptr; + const char* const end; + const Formatter::Arg* const* args; + const int args_count; + int num_args; + int cur_arg; +}; + +struct Item { + Item() + : fill(' '), + space_pad(false), + truncate(false), + width(0), + extra_width(0), + precision(6), + flags(ios_base::dec) { + } + + char fill; + bool space_pad; + bool truncate; + int width; + int extra_width; + int precision; + ios_base::fmtflags flags; +}; + +void PutItem(State* s, const Item& i) { + // TODO(pmattis): Handle positional arguments. + const int width = + (i.width >= 0) ? i.width : s->args[s->cur_arg++]->AsInt(); + const int precision = + (i.precision >= 0) ? i.precision : s->args[s->cur_arg++]->AsInt(); + + if (i.truncate || i.space_pad) { + // Truncation or space padding required, output to a string and + // truncate the result. + const int max_width = + (i.truncate ? precision + i.extra_width : 0); + std::ostringstream ss; + ss.fill(i.fill); + ss.width(max_width); + ss.precision(6); + ss.flags((i.flags & ~ios_base::adjustfield) | + ios_base::internal); + s->args[s->cur_arg++]->Put(ss); + s->os.fill(' '); + s->os.width(width); + s->os.flags(i.flags); + const string& str = ss.str(); + if (i.space_pad && !str.empty() && str[0] == '+') { + s->os.write(" ", 1); + s->os << str.substr(1, max_width - 1); + } else { + s->os << str.substr(0, max_width); + } + } else { + // No truncation required, output directly to stream. + s->os.fill(i.fill); + s->os.width(width); + s->os.precision(precision); + s->os.flags(i.flags); + s->args[s->cur_arg++]->Put(s->os); + } +} + +const char* ParseInt(const char* start, int *value) { + char* end; + *value = strtol(start, &end, 10); + return end; +} + +bool ProcessDirective(State* s) { + const char* const start = s->ptr; + bool parsed_precision = false; + Item i; + + // Parse flags. + while (++s->ptr < s->end) { + switch (*s->ptr) { + case '#': + i.flags |= ios_base::showbase; + i.flags |= ios_base::showpoint; + i.flags |= ios_base::boolalpha; + break; + case '0': + i.fill = '0'; + break; + case '-': + i.flags &= ~ios_base::adjustfield; + i.flags |= ios_base::left; + break; + case '_': + i.flags &= ~ios_base::adjustfield; + i.flags |= ios_base::internal; + break; + case ' ': + if (!(i.flags & ios_base::showpos)) { + i.space_pad = true; + i.flags |= ios_base::showpos; + } + break; + case '+': + i.space_pad = false; + i.flags |= ios_base::showpos; + break; + default: + goto parse_width; + } + } + goto error; + +parse_width: + if (*s->ptr == '*') { + i.width = -1; + ++s->ptr; + } else if (isdigit(*s->ptr)) { + s->ptr = ParseInt(s->ptr, &i.width); + if (s->ptr >= s->end) { + goto error; + } + } + + // parse_precision: + if (*s->ptr == '.') { + if (++s->ptr >= s->end) { + goto error; + } + if (*s->ptr == '*') { + i.precision = -1; + ++s->ptr; + } else if (isdigit(*s->ptr)) { + s->ptr = ParseInt(s->ptr, &i.precision); + if (s->ptr >= s->end) { + goto error; + } + } + parsed_precision = true; + } + + // done: + switch (*s->ptr) { + case 'u': + i.space_pad = false; + i.flags &= ~ios_base::showpos; + case 'd': + case 'i': + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::dec; + if (parsed_precision) { + i.truncate = true; + i.fill = '0'; + if (i.flags & ios_base::showpos) { + i.extra_width = 1; + } + } + break; + + case 'X': + i.flags |= ios_base::uppercase; + case 'p': + case 'x': + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::hex; + if (parsed_precision) { + i.truncate = true; + i.fill = '0'; + if (i.flags & ios_base::showbase) { + i.extra_width = 2; + } + } + break; + + case 'o': + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::oct; + if (parsed_precision) { + i.truncate = true; + i.fill = '0'; + if (i.flags & ios_base::showbase) { + i.extra_width = 1; + } + } + break; + + case 'f': + i.flags &= ~ios_base::floatfield; + i.flags |= ios_base::fixed; + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::dec; + break; + + case 'E': + i.flags |= ios_base::uppercase; + case 'e': + i.flags &= ~ios_base::floatfield; + i.flags |= ios_base::scientific; + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::dec; + break; + + case 'G': + i.flags |= ios_base::uppercase; + case 'g': + i.flags &= ~ios_base::floatfield; + i.flags &= ~ios_base::basefield; + i.flags |= ios_base::dec; + break; + + case 'C': + case 'c': + i.truncate = true; + i.precision = 1; + break; + + case 'S': + case 's': + case '@': + if (parsed_precision) { + i.truncate = true; + } + break; + + case 'n' : + // TODO(pmattis) + break; + + default: + goto error; + } + + if (i.flags & ios_base::left) { + if (!i.truncate) { + // Left alignment (pad on right) and no truncation specification implies + // pad with spaces. + i.fill = ' '; + } + } else if (i.fill != ' ') { + // Non-left alignment and non-space padding, convert to internal padding. + i.flags = i.flags & ~ios_base::adjustfield; + i.flags |= ios_base::internal; + } + + ++s->ptr; + + s->num_args += 1 + (i.width == -1) + (i.precision == -1); + if (s->num_args > s->args_count) { + return false; + } + + PutItem(s, i); + return true; + +error: +#ifdef DEBUG + DIE("Error: unterminated format: ", Slice(start, s->ptr - start)); +#else // DEBUG + s->os << "ptr - start) << "'>"; +#endif // DEBUG + return false; +} + +} // namespace + +const FormatMaker& Format = *(new FormatMaker); + +void Formatter::Apply( + ostream& os, const Arg* const* args, int args_count) const { + State s(os, format_, args, args_count); + const char* last = s.ptr; + while (s.ptr < s.end) { + s.ptr = std::find(s.ptr, s.end, '%'); + if (s.ptr == s.end) { + break; + } + os.write(last, s.ptr - last); + if (&s.ptr[1] < s.end && s.ptr[1] == '%') { + last = s.ptr + 1; + s.ptr += 2; + continue; + } + if (!ProcessDirective(&s)) { + break; + } + last = s.ptr; + } + + if (s.args_count != s.num_args) { +#ifdef DEBUG + DIE("Error: incorrect number of format arguments: %d != %d", + s.args_count, s.num_args); +#else // DEBUG + os << ""; +#endif // DEBUG + return; + } + + os.write(last, s.end - last); +} + +string Formatter::DebugString(const Arg* const* args, int args_count) const { + std::ostringstream ss; + ss << format_; + for (int i = 0; i < args_count; ++i) { + ss << " % "; + args[i]->Put(ss); + } + return ss.str(); +} diff --git a/clients/shared/Format.h b/clients/shared/Format.h new file mode 100644 index 0000000..11468c2 --- /dev/null +++ b/clients/shared/Format.h @@ -0,0 +1,363 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_FORMAT_H +#define VIEWFINDER_FORMAT_H + +#import +#import +#import +#import "Utils.h" +#import "StringUtils.h" + +class Formatter { + enum { + kSize = 0 + }; + + public: + // A single argument to the formatter. Stores a pointer to the argument and + // provides methods to output the typed argument to an ostream and to + // retrieve its value as an int. + class Arg { + typedef void (*PutType)(ostream& os, const void* val); + typedef int (*AsIntType)(const void* val); + + template + struct Helper { + static void Put(ostream& os, const T* val) { + os << *val; + } + +#ifdef __OBJC__ + static void PutValue(ostream& os, T val) { + os << val; + } +#endif // __OBJC__ + + static int AsInt(const T* val) { + return ExtractInt(*val); + } + static int Zero(const T* val) { + return 0; + } + + static int ExtractInt(char val) { return val; }; + static int ExtractInt(unsigned char val) { return val; }; + static int ExtractInt(short val) { return val; }; + static int ExtractInt(unsigned short val) { return val; }; + static int ExtractInt(int val) { return val; }; + static int ExtractInt(unsigned int val) { return val; }; + static int ExtractInt(long val) { return val; }; + static int ExtractInt(unsigned long val) { return val; }; + static int ExtractInt(long long val) { return val; }; + static int ExtractInt(unsigned long long val) { return val; }; + static int ExtractInt(float val) { return static_cast(val); }; + static int ExtractInt(double val) { return static_cast(val); }; + static int ExtractInt(long double val) { return static_cast(val); }; + template static int ExtractInt(const Q& val) { return 0; }; + }; + + public: + template + Arg(const T& v) + : val_(&v), + put_(reinterpret_cast(&Helper::Put)), + as_int_(reinterpret_cast(&Helper::AsInt)) { + } + template + Arg(const volatile T& v) + // Somewhat confusingly, const_cast is used to cast about volatile. + : val_(const_cast(&v)), + put_(reinterpret_cast(&Helper::Put)), + as_int_(reinterpret_cast(&Helper::AsInt)) { + } + +#ifdef __OBJC__ + Arg(id v) + : val_((__bridge const void*) v), + put_(reinterpret_cast(&Helper::PutValue)), + as_int_(reinterpret_cast(&Helper::Zero)) { + } + Arg(NSString* v) + : val_((__bridge const void*) v), + put_(reinterpret_cast(&Helper::PutValue)), + as_int_(reinterpret_cast(&Helper::Zero)) { + } + Arg(NSData* v) + : val_((__bridge const void*) v), + put_(reinterpret_cast(&Helper::PutValue)), + as_int_(reinterpret_cast(&Helper::Zero)) { + } +#endif // __OBJC__ + + void Put(ostream& os) const { + put_(os, val_); + } + + int AsInt() const { + return as_int_(val_); + } + + private: + const void* const val_; + const PutType put_; + const AsIntType as_int_; + }; + + // A node in the list of arguments to the formatter. Contains a single + // argument and a reference to the tail of the list. The tail of the list is + // the Format object itself. + template + class ArgList { + public: + enum { + kIndex = Tail::kSize, + kSize = 1 + kIndex, + }; + + public: + ArgList(const Arg& a, const Tail& t) + : arg_(a), + tail_(t) { + } + + // Returns a new list with the specified argument prepended (i.e. argument + // index 0 is at the end of the list). + ArgList > operator%(const Arg& arg) const { + return ArgList >(arg, *this); + } + + // Applies the arguments in the list to the associated format object, + // outputting the result to the ostream. + void Apply(ostream& os) const { + Formatter::Arg const* array[kSize]; + Fill(array)->Apply(os, array, kSize); + } + + // Applies the arguments in the list to the associated format object, + // outputting the result to a string. + string ToString() const { + std::ostringstream ss; + ss << *this; + return ss.str(); + } + + // Returns a string containing the format string and all of the arguments. + string DebugString() const { + Arg const* array[kSize]; + return Fill(array)->DebugString(array, kSize); + } + + // Fills the specified array with pointers to the arguments. + const Formatter* Fill(Arg const** array) const { + array[kIndex] = &arg_; + return tail_.Fill(array); + } + + // String conversion operator. + operator string() const { + return ToString(); + } + +#ifdef __OBJC__ + NSString* ToNSString() const { + return NewNSString(ToString()); + } + operator NSString*() const { + return ToNSString(); + } +#endif // __OBJC__ + + private: + const Arg& arg_; + const Tail& tail_; + }; + + public: + explicit Formatter(const string& str) + : format_(str) { + } + + ArgList operator%(const Arg& arg) const { + return ArgList(arg, *this); + } + + // Outputs the format object to the ostream. An error will occur if the + // format string requires any arguments. + void Apply(ostream& os) const { + Apply(os, NULL, 0); + } + + // Applies the array of arguments to the format object, outputting the result + // to the ostream. + void Apply(ostream& os, const Arg* const* args, int args_count) const; + + // Outputs the format object to a string. An error will occur if the format + // string requires any arguments. + string ToString() const { + std::ostringstream ss; + Apply(ss, NULL, 0); + return ss.str(); + } + + // Returns the format string. + const string& DebugString() const { + return format_; + } + + // String conversion operator. + operator string() const { + return ToString(); + } + +#ifdef __OBJC__ + operator NSString*() const { + return ToNSString(); + } + NSString* ToNSString() const { + return NewNSString(ToString()); + } +#endif // __OBJC__ + + private: + // Internal method for generating the debug string using the specified array + // of arguments. + string DebugString(const Arg* const* args, int args_count) const; + + // The Formatter object is the tail of the argument list. This is a required + // method to be compatible with ArgList. + const Formatter* Fill(Arg const** array) const { return this; } + + private: + const string format_; +}; + +struct FormatMaker { + typedef Formatter::Arg Arg; + + FormatMaker() { + } + explicit FormatMaker(const string& s) + : str(s) { + } + + Formatter operator()(const char* fmt) const { + return Formatter(fmt); + } + + FormatMaker operator()(const char* fmt, const Arg& a0) const { + std::ostringstream ss; + const Arg* const args[] = { &a0 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + string operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + FormatMaker operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9) const { + std::ostringstream ss; + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8, + &a9 }; + Formatter(fmt).Apply(ss, args, ARRAYSIZE(args)); + return FormatMaker(ss.str()); + } + + operator string() const { + return str; + } + +#ifdef __OBJC__ + operator NSString*() const { + return NewNSString(str); + } +#endif // __OBJC__ + + string str; +}; + +extern const FormatMaker& Format; + +inline ostream& operator<<(ostream& os, const FormatMaker& format) { + os << format.str; + return os; +} + +inline ostream& operator<<(ostream& os, const Formatter& format) { + format.Apply(os); + return os; +} + +template +inline ostream& operator<<(ostream& os, const Formatter::ArgList& args) { + args.Apply(os); + return os; +} + +#endif // VIEWFINDER_FORMAT_H diff --git a/clients/shared/FullTextIndex.cc b/clients/shared/FullTextIndex.cc new file mode 100644 index 0000000..26e6863 --- /dev/null +++ b/clients/shared/FullTextIndex.cc @@ -0,0 +1,803 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#import +#import +#import "AppState.h" +#import "AsyncState.h" +#import "DBFormat.h" +#import "FullTextIndex.h" +#import "FullTextIndexInternal.h" +#import "FullTextIndexMetadata.pb.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "StringUtils.h" + +namespace { + +// Parse everything between unicode separator characters. This +// will include all punctuation, both internal to the string and +// leading and trailing. +LazyStaticPtr kWhitespaceUnicodeRE = { "([\\pZ]+)" }; +LazyStaticPtr kNonAlphaNumUnicodeRE = { "[^\\pL\\pN]+" }; + +// Indexed terms are stored with the following format: +// +// ft//i/ +// +// The database value is an empty string; all the information is contained in the key. +LazyStaticPtr kIndexTermKeyRE = { "ft/[a-z]+/i/(\\d+)\t([^\t]*)\t(.*)" }; + +// Previous version of the index term RE used for backwards compatibility. +LazyStaticPtr kIndexTermKeyREv1 = { "ft/[a-z]+/i/(\\d+)\t([^\t]*)\t\\d+\t(.*)" }; + +const char* kIndexTermKeyFormat = "%s%s\t%s\t%s"; + +// Lexicon terms are stored with the following format: +// +// ft//l/ (|'') +// +// The constituent pieces are tab-delimited because we want to allow +// arbitrary punctuation and symbols in names. Think hyphenation, +// apostrophes, periods (possibly slashes '/', which was the original +// delimiter). Special tokens may also include spaces, although these +// tokens may not be queried in the usual manner. +// +// The database value is a FullTextLexiconMetadata protobuf. +LazyStaticPtr kLexiconKeyRE = { "ft/[a-z]+/l/([^\t]+)\t([^\t]*)" }; +const char* kLexiconKeyFormat = "%s%s\t%s"; + +// Reverse lexicon entries are stored with the following format: +// +// ft//r/ +// +// The value is the entire database key of the corresponding lexicon entry (ft/*/l/*). +const char* kReverseLexiconKeyFormat = "ft/%s/r/%s"; + +// Metadata entries are stored under ft//m/. +const char* kMetadataKeyFormat = "ft/%s/m/%s"; + +// Lexicon invalidation keys are stored under ft//ti/. +const char* kTokenInvalidationPrefixFormat = "ft/%s/ti/"; +LazyStaticPtr kTokenInvalidationKeyRE = { "ft/[a-z]+/ti/(\\d+)" }; +const char* kTokenInvalidationKeyFormat = "ft/%s/ti/%s"; + +const int kLexiconCacheSize = 1000; + +// Format used to build filter regexp (case-insensitve match) on the filter +// string or on the filter string alone or with a leading separator character. +const char* kFilterREFormat = "(?i)(?:^|[\\s]|[[:punct:]])(%s)"; + +const DBRegisterKeyIntrospect kFullTextIndexKeyIntrospect( + DBFormat::full_text_index_key(), NULL, NULL); + +bool operator<(const FullTextResultIterator& a, const FullTextResultIterator& b) { + if (a.sort_key() != b.sort_key()) { + return a.sort_key() < b.sort_key(); + } + return a.doc_id() < b.doc_id(); +} + +// Compares two iterator *pointers*. Useful when constructing a heap of iterators. +struct ResultIteratorGreaterThan { + bool operator()(FullTextResultIterator* a, FullTextResultIterator* b) { + return *b < *a; + } +}; + +} // namespace + + +FullTextQueryIteratorBuilder::FullTextQueryIteratorBuilder( + std::initializer_list indexes, const DBHandle& db) + : indexes_(indexes), + db_(db) { +} + +FullTextQueryIteratorBuilder::~FullTextQueryIteratorBuilder() { +} + +FullTextResultIterator* FullTextQueryIteratorBuilder::BuildIterator(const FullTextQuery& query) { + stack_.push_back(Accumulator()); + VisitNode(query); + CHECK_EQ(stack_.size(), 1); + CHECK_EQ(stack_[0].size(), 1); + return stack_[0][0]; +} + +void FullTextQueryIteratorBuilder::VisitTermNode(const FullTextQueryTermNode& node) { + vector iterators; + for (auto it : indexes_) { + iterators.push_back(it->CreateTokenIterator(db_, node.term())); + } + stack_.back().push_back(full_text_index::OrResultIterator::Create(iterators)); +} + +void FullTextQueryIteratorBuilder::VisitPrefixNode(const FullTextQueryPrefixNode& node) { + vector iterators; + for (auto it : indexes_) { + iterators.push_back(it->CreateTokenPrefixIterator(db_, node.prefix())); + } + stack_.back().push_back(full_text_index::OrResultIterator::Create(iterators)); +} + +void FullTextQueryIteratorBuilder::VisitParentNode(const FullTextQueryParentNode& node) { + stack_.push_back(Accumulator()); + VisitChildren(node); + FullTextResultIterator* new_iter; + if (node.type() == FullTextQuery::AND) { + new_iter = full_text_index::AndResultIterator::Create(stack_.back()); + } else { + new_iter = full_text_index::OrResultIterator::Create(stack_.back()); + } + stack_.pop_back(); + stack_.back().push_back(new_iter); +} + + +FullTextQuery::~FullTextQuery() { +} + +FullTextQuery* FullTextQuery::Parse(const Slice& query, int options) { + // Break the incoming query into terms at whitespace boundaries. + const vector words = SplitWords(query); + vector nodes; + for (int i = 0; i < words.size(); i++) { + if (options & PREFIX_MATCH) { + nodes.push_back(new FullTextQueryPrefixNode(ToLowercase(words[i]))); + } else { + nodes.push_back(new FullTextQueryTermNode(ToLowercase(words[i]))); + } + } + return new FullTextQueryAndNode(nodes); +} + + +FullTextQueryTermNode::FullTextQueryTermNode(const Slice& term) + : term_(term.as_string()) { +} + +FullTextQueryTermNode::~FullTextQueryTermNode() { +} + + +FullTextQueryPrefixNode::FullTextQueryPrefixNode(const Slice& prefix) + : prefix_(prefix.as_string()) { +} + +FullTextQueryPrefixNode::~FullTextQueryPrefixNode() { +} + + +FullTextQueryParentNode::FullTextQueryParentNode(const vector& children) + : children_(children) { +} + +FullTextQueryParentNode::~FullTextQueryParentNode() { +} + +string FullTextQueryParentNode::ToString() const { + string s = "("; + s.append(type() == AND ? "and" : "or"); + for (auto child : children()) { + s.append(" "); + s.append(child->ToString()); + } + s.append(")"); + return s; +} + +FullTextQueryVisitor::~FullTextQueryVisitor() { +} + +void FullTextQueryVisitor::VisitNode(const FullTextQuery& node) { + switch (node.type()) { + case FullTextQuery::TERM: + VisitTermNode(static_cast(node)); + break; + case FullTextQuery::AND: + VisitAndNode(static_cast(node)); + break; + case FullTextQuery::OR: + VisitOrNode(static_cast(node)); + break; + case FullTextQuery::PREFIX: + VisitPrefixNode(static_cast(node)); + break; + } +} + +void FullTextQueryVisitor::VisitChildren(const FullTextQueryParentNode& node) { + for (FullTextQuery *const child : node.children()) { + VisitNode(*child); + } +} + +void FullTextQueryVisitor::VisitParentNode(const FullTextQueryParentNode& node) { + VisitChildren(node); +} + +void FullTextQueryVisitor::VisitAndNode(const FullTextQueryAndNode& node) { + VisitParentNode(node); +} + +void FullTextQueryVisitor::VisitOrNode(const FullTextQueryOrNode& node) { + VisitParentNode(node); +} + + +FullTextResultIterator::~FullTextResultIterator() { +} + +void FullTextResultIterator::Seek(const FullTextResultIterator& other) { + while (Valid() && *this < other) { + Next(); + } +} + +FullTextIndexTerm::FullTextIndexTerm() + : index(0) { +} + +FullTextIndexTerm::FullTextIndexTerm(const string& it, const string& rt, int i) + : index_term(it), + raw_term(it == rt ? "" : rt), + index(i) { +} + +FullTextIndexTerm::~FullTextIndexTerm() { +} + +FullTextIndex::FullTextIndex(AppState* state, const Slice& name) + : state_(state), + name_(name.as_string()), + index_prefix_(DBFormat::full_text_index_key(name_) + "i/"), + lexicon_prefix_(DBFormat::full_text_index_key(name_) + "l/"), + updating_lexicon_stats_(false) { + MaybeUpdateLexiconStats(); +} + +FullTextIndex::~FullTextIndex() { +} + +FullTextResultIterator* FullTextIndex::CreateTokenIterator(const DBHandle& db, const Slice& token) const { + return CreateTokenPrefixIterator(db, token.as_string() + "\t"); +} + +FullTextResultIterator* FullTextIndex::CreateTokenPrefixIterator(const DBHandle& db, const Slice& token_prefix) const { + vector token_iters; + for (DB::PrefixIterator lex_iter(db, lexicon_prefix_ + token_prefix.as_string()); + lex_iter.Valid(); + lex_iter.Next()) { + const Slice lex_key = lex_iter.key(); + + Slice index_term; + Slice raw_term; + if (!RE2::FullMatch(lex_key, *kLexiconKeyRE, &index_term, &raw_term)) { + LOG("index: unable to parse lexicon key: %s", lex_key); + continue; + } + + FullTextLexiconMetadata lex_data; + if (!lex_data.ParseFromArray(lex_iter.value().data(), lex_iter.value().size())) { + LOG("index: unable to parse lexicon value for %s", lex_key); + continue; + } + + // If the raw term differs from the filter term, save the matching prefix from the raw term. + string raw_prefix; + if (!raw_term.empty() && raw_term != index_term) { + raw_prefix = FindRawPrefix(token_prefix, raw_term); + } + token_iters.push_back(new full_text_index::TokenResultIterator(*this, db, lex_data.token_id(), raw_prefix)); + } + + return full_text_index::OrResultIterator::Create(token_iters); +} + +FullTextResultIterator* FullTextIndex::Search(const DBHandle& db, const FullTextQuery& query) const { + FullTextQueryIteratorBuilder builder({this}, db); + return builder.BuildIterator(query); +} + +void FullTextIndex::GetSuggestions(const DBHandle& db, const Slice& prefix, SuggestionResults* results) { + for (DB::PrefixIterator iter(db, lexicon_prefix_ + prefix.as_string()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + Slice index_term; + Slice raw_term; + if (!RE2::FullMatch(key, *kLexiconKeyRE, &index_term, &raw_term)) { + LOG("index: unable to parse lexicon key: %s", key); + continue; + } + FullTextLexiconMetadata data; + if (!data.ParseFromArray(iter.value().data(), iter.value().size())) { + LOG("index: unable to parse lexicon data for: %s", key); + } + if (data.count() == 0) { + continue; + } + results->push_back(std::make_pair(data.count(), + (raw_term.empty() ? index_term : raw_term).as_string())); + } + std::sort(results->begin(), results->end()); + std::reverse(results->begin(), results->end()); +} + +string FullTextIndex::FindRawPrefix(const Slice& index_prefix, const Slice& raw_term) { + // The matching prefix can be tricky. We walk the raw term until + // we've assembled the same number of alphanumeric characters as + // were contained in the filter term. + int target_len = 0; + for (UnicodeCharIterator it(index_prefix); !it.Done(); it.Advance()) { + if (IsAlphaNumUnicode(it.Get())) { + ++target_len; + } + } + + icu::UnicodeString out; + for (UnicodeCharIterator it(raw_term); !it.Done() && target_len > 0; it.Advance()) { + UChar32 c = it.Get(); + out.append(c); + if (IsAlphaNumUnicode(c)) { + --target_len; + } + } + string out_utf8; + out.toUTF8String(out_utf8); + return out_utf8; +} + + +int FullTextIndex::ParseIndexTerms(int index, const Slice& phrase, + vector* terms) const { + const vector words = SplitWords(phrase); + for (int i = 0; i < words.size(); i++) { + DenormalizeIndexTerm(index, ToLowercase(words[i]), terms); + index++; + } + return index; +} + +// Generates additional index terms by lossily converting to 7-bit +// ascii. Adds index term and any denormalized version(s) of it to +// 'terms'. +void FullTextIndex::DenormalizeIndexTerm(int index, const string& term, + vector* terms) const { + //LOG("pushing index term %s", term); + terms->push_back(FullTextIndexTerm(term, term, index)); + + const string unpunctuated = RemovePunctuation(term); + if (!unpunctuated.empty() && unpunctuated != term) { + terms->push_back(FullTextIndexTerm(unpunctuated, term, index)); + } + + // See if converting to ascii yields a different, but + // non-empty result. If so, index that as well. + string lossy(ToAsciiLossy(term)); + // The transliterator may have introduced spaces (especially for Chinese, where it adds a space + // between syllables), but we don't allow spaces in tokens. + RE2::GlobalReplace(&lossy, *kWhitespaceUnicodeRE, ""); + if (!lossy.empty() && term != lossy) { + terms->push_back(FullTextIndexTerm(lossy, term, index)); + string np_lossy = ToAsciiLossy(unpunctuated); + RE2::GlobalReplace(&np_lossy, *kWhitespaceUnicodeRE, ""); + if (!np_lossy.empty() && np_lossy != unpunctuated) { + terms->push_back(FullTextIndexTerm(np_lossy, term, index)); + } + } +} + +int FullTextIndex::AddVerbatimToken(int index, const Slice& token, + vector* terms) const { + const string token_str(token.as_string()); + terms->push_back(FullTextIndexTerm(token_str, token_str, index)); + return index + 1; +} + +void FullTextIndex::UpdateIndex(const vector& terms, + const Slice& doc_id, const Slice& sort_key, + google::protobuf::RepeatedPtrField* disk_terms, + const DBHandle& updates) { + // TODO(ben): don't remove and re-add terms that were present before and after. + // It's inefficient and causes token ids to be wasted (a token loses its id when its refcount hits zero). + + // Remove any existing name indexes. + RemoveTerms(disk_terms, updates); + disk_terms->Clear(); + + CHECK_EQ(sort_key.find('\t'), Slice::npos); + + // Add all the indexed terms. + for (int i = 0; i < terms.size(); ++i) { + const int64_t token_id = AddToLexicon(terms[i]); + const string term_key = Format(kIndexTermKeyFormat, index_prefix_, token_id, + sort_key, doc_id); + *disk_terms->Add() = term_key; + updates->Put(term_key, ""); + InvalidateLexiconStats(token_id, updates); + } +} + +void FullTextIndex::RemoveTerms(google::protobuf::RepeatedPtrField* disk_terms, + const DBHandle& updates) { + for (int i = 0; i < disk_terms->size(); ++i) { + const string& key = disk_terms->Get(i); + updates->Delete(key); + + int64_t token_id; + Slice sort_key; + Slice doc_id; + // NOTE(ben): if this doesn't match, we're either migrating from a + // pre-lexicon format or the format has changed. In the later + // case the lexicon counts may get out of sync, so we'll need to either + // rebuild the lexicon from scratch or ensure we can continue to parse + // the old keys here. + if (RE2::FullMatch(key, *kIndexTermKeyRE, &token_id, &sort_key, &doc_id) || + RE2::FullMatch(key, *kIndexTermKeyREv1, &token_id, &sort_key, &doc_id)) { + InvalidateLexiconStats(token_id, updates); + } + } +} + +string FullTextIndex::TimestampSortKey(WallTime time) { + int64_t usec = time * 1000000; + string s; + OrderedCodeEncodeVarint64Decreasing(&s, usec); + // Sort keys cannot contain spaces, so base64hex-encode it. + return Base64HexEncode(s, false); +} + +string FullTextIndex::RemovePunctuation(const Slice& term) { + string unpunctuated = term.as_string(); + RE2::GlobalReplace(&unpunctuated, *kNonAlphaNumUnicodeRE, ""); + return unpunctuated; +} + +void FullTextIndex::DrainBackgroundOps() { + MutexLock lock(&lexicon_stats_mutex_); + lexicon_stats_mutex_.Wait([this]{ + return !updating_lexicon_stats_; + }); +} + +string FullTextIndex::FormatIndexTermPrefix(int64_t token_id) const { + // Append a tab because we don't want token ids that are prefixes of + // each other, just records that have the token id as a prefix. + return Format("%s%s\t", index_prefix_, token_id); +} + +RE2* FullTextIndex::BuildFilterRE(const StringSet& all_terms) { + string s; + for (StringSet::const_iterator iter(all_terms.begin()); + iter != all_terms.end(); + ++iter) { + if (!s.empty()) { + s += "|"; + } + const string& t = *iter; + if (!t.empty()) { + s += RE2::QuoteMeta(*iter); + } + } + if (s.empty()) { + return NULL; + } else { + // We want the regexp to return the longest match so that the regexp + // "kat|k" will match "[kat]hryn" and not "[k]athryn". + RE2::Options opts; + opts.set_longest_match(true); + return new RE2(string(Format(kFilterREFormat, s)), opts); + } +} + +string FullTextIndex::FormatLexiconKey(const Slice& term, const Slice& raw_term) const { + return Format(kLexiconKeyFormat, lexicon_prefix_, term, raw_term); +} + +int64_t FullTextIndex::AddToLexicon(const FullTextIndexTerm& term) { + const string lex_key = FormatLexiconKey(term.index_term, term.raw_term); + MutexLock lock(&lexicon_mutex_); + + const int64_t* cache_token_id = FindPtrOrNull(lexicon_cache_, lex_key); + if (cache_token_id) { + return *cache_token_id; + } + + // The mapping of tokens to ids is append-only, so write it immediately in a separate transaction. + DBHandle updates = state_->NewDBTransaction(); + FullTextLexiconMetadata data; + const bool exists = updates->GetProto(lex_key, &data); + if (!exists) { + const string id_key = Format(kMetadataKeyFormat, name_, "next_id"); + int64_t id = updates->Get(id_key); + updates->Put(id_key, id + 1); + updates->Put(Format(kReverseLexiconKeyFormat, name_, id), lex_key); + data.set_token_id(id); + updates->PutProto(lex_key, data); + } + updates->Commit(); + + if (lexicon_cache_.size() > kLexiconCacheSize) { + // TODO(ben): Use LRU eviction instead of throwing the whole thing out. + lexicon_cache_.clear(); + } + lexicon_cache_[lex_key] = data.token_id(); + + return data.token_id(); +} + +void FullTextIndex::InvalidateLexiconStats(int64_t token_id, const DBHandle& updates) { + const string invalidation_key = Format(kTokenInvalidationKeyFormat, name_, token_id); + updates->Put(invalidation_key, ""); + CHECK(updates->AddCommitTrigger( + Format("InvalidateTokenStats:%s", name_), + [this]{ + MaybeUpdateLexiconStats(); + })); +} + +int64_t FullTextIndex::AllocateTokenIdLocked(const Slice& lex_key, const DBHandle& updates) { + lexicon_mutex_.AssertHeld(); + const string key = Format(kMetadataKeyFormat, name_, "next_id"); + int64_t id = updates->Get(key); + updates->Put(key, id + 1); + updates->Put(Format(kReverseLexiconKeyFormat, name_, id), lex_key); + FullTextLexiconMetadata data; + data.set_token_id(id); + updates->PutProto(lex_key, data); + return id; +} + +void FullTextIndex::MaybeUpdateLexiconStats() { + MutexLock lock(&lexicon_stats_mutex_); + if (updating_lexicon_stats_) { + return; + } + updating_lexicon_stats_ = true; + state_->async()->dispatch_background([this]{ + while (UpdateLexiconStats()) { + } + // There's a small race condition here if a transaction commits between the last UpdateLexiconStats + // call and our setting updating_lexicon_stats, but it's OK if stats are a bit out of date. + MutexLock lock(&lexicon_stats_mutex_); + updating_lexicon_stats_ = false; + }); +} + +bool FullTextIndex::UpdateLexiconStats() { + DBHandle updates = state_->NewDBTransaction(); + for (DB::PrefixIterator inv_iter(updates, (string)Format(kTokenInvalidationPrefixFormat, name_)); + inv_iter.Valid(); + inv_iter.Next()) { + int64_t token_id; + if (!RE2::FullMatch(inv_iter.key(), *kTokenInvalidationKeyRE, &token_id)) { + LOG("index: could not parse token invalidation key %s", inv_iter.key()); + DCHECK(false); + continue; + } + + const string rev_lex_key = Format(kReverseLexiconKeyFormat, name_, token_id); + const string lex_key = updates->Get(rev_lex_key); + if (lex_key.empty()) { + LOG("index: could not find lexicon key for token %s", rev_lex_key); + continue; + } + + int hit_count = 0; + for (DB::PrefixIterator hit_iter(updates, FormatIndexTermPrefix(token_id)); + hit_iter.Valid(); + hit_iter.Next()) { + hit_count++; + } + + FullTextLexiconMetadata lex_data; + if (!updates->GetProto(lex_key, &lex_data)) { + LOG("index: could not load lexicon stats for %s", lex_key); + continue; + } + lex_data.set_count(hit_count); + updates->PutProto(lex_key, lex_data); + + updates->Delete(inv_iter.key()); + } + const bool changed = updates->tx_count(); + updates->Commit(); + return changed; +} + +namespace full_text_index { + +NullResultIterator::NullResultIterator() { +} + +NullResultIterator::~NullResultIterator() { +} + +FullTextResultIterator* AndResultIterator::Create(const vector& iterators) { + if (iterators.size() == 0) { + return new NullResultIterator(); + } else if (iterators.size() == 1) { + return iterators[0]; + } else { + return new AndResultIterator(iterators); + } +} + +AndResultIterator::AndResultIterator(const vector& iterators) + : valid_(iterators.size() > 0), + iterators_(iterators) { + SynchronizeIterators(); +} + +AndResultIterator::~AndResultIterator() { + Clear(&iterators_); +} + +bool AndResultIterator::Valid() const { + return valid_; +} + +void AndResultIterator::Next() { + DCHECK(valid_); + // Precondition: all iterators are pointing to the same document. + // Increment one then advance until they align again. + iterators_[0]->Next(); + SynchronizeIterators(); +} + +const Slice AndResultIterator::doc_id() const { + DCHECK(valid_); + return iterators_[0]->doc_id(); +} + +const Slice AndResultIterator::sort_key() const { + DCHECK(valid_); + return iterators_[0]->sort_key(); +} + +void AndResultIterator::GetRawTerms(StringSet* raw_terms) const { + for (int i = 0; i < iterators_.size(); i++) { + iterators_[0]->GetRawTerms(raw_terms); + } +} + +void AndResultIterator::SynchronizeIterators() { + if (iterators_.size() == 0 || + !iterators_[0]->Valid()) { + valid_ = false; + return; + } + // Try to bring the rest of the iterators to match the first one. + for (int i = 1; i < iterators_.size(); i++) { + iterators_[i]->Seek(*iterators_[0]); + if (!iterators_[i]->Valid()) { + valid_ = false; + return; + } + // The iterator we just advanced overshot the first iterator. + // Bring the first one up to match and start over. + if (*iterators_[0] < *iterators_[i]) { + iterators_[0]->Seek(*iterators_[i]); + if (!iterators_[0]->Valid()) { + valid_ = false; + return; + } + i = 0; + } + } +} + +FullTextResultIterator* OrResultIterator::Create(const vector& iterators) { + if (iterators.size() == 0) { + return new NullResultIterator(); + } else if (iterators.size() == 1) { + return iterators[0]; + } else { + return new OrResultIterator(iterators); + } +} + +OrResultIterator::OrResultIterator(const vector& iterators) { + for (int i = 0; i < iterators.size(); i++) { + if (iterators[i]->Valid()) { + iterators_.push_back(iterators[i]); + } else { + delete iterators[i]; + } + } + std::make_heap(iterators_.begin(), iterators_.end(), ResultIteratorGreaterThan()); +} + +OrResultIterator::~OrResultIterator() { + Clear(&iterators_); +} + +bool OrResultIterator::Valid() const { + return !iterators_.empty(); +} + +void OrResultIterator::Next() { + if (iterators_.size() == 0) { + return; + } + const string current_doc_id = iterators_[0]->doc_id().as_string(); + while (iterators_.size() > 0 && + iterators_[0]->doc_id() == current_doc_id) { + FullTextResultIterator* iter = iterators_[0]; + std::pop_heap(iterators_.begin(), iterators_.end(), ResultIteratorGreaterThan()); + iter->Next(); + if (iter->Valid()) { + std::push_heap(iterators_.begin(), iterators_.end(), ResultIteratorGreaterThan()); + } else { + delete iter; + iterators_.resize(iterators_.size() - 1); + } + } +} + +const Slice OrResultIterator::doc_id() const { + return iterators_[0]->doc_id(); +} + +const Slice OrResultIterator::sort_key() const { + return iterators_[0]->sort_key(); +} + +void OrResultIterator::GetRawTerms(StringSet* raw_terms) const { + for (int i = 0; i < iterators_.size(); i++) { + // Get the raw terms from all iterators that match the current position. + if (*iterators_[0] < *iterators_[i]) { + continue; + } + iterators_[0]->GetRawTerms(raw_terms); + } +} + +TokenResultIterator::TokenResultIterator(const FullTextIndex& index, const DBHandle& db, + int64_t token_id, const Slice& raw_prefix) + : token_id_(token_id), + raw_prefix_(raw_prefix.as_string()), + db_iter_(db, index.FormatIndexTermPrefix(token_id_)), + error_(false) { + ParseHit(); +} + +TokenResultIterator::~TokenResultIterator() { +} + +bool TokenResultIterator::Valid() const { + return !error_ && db_iter_.Valid(); +} + +void TokenResultIterator::Next() { + db_iter_.Next(); + ParseHit(); +} + +void TokenResultIterator::ParseHit() { + if (!db_iter_.Valid()) { + return; + } + const Slice key = db_iter_.key(); + int64_t token_id; + if (!RE2::FullMatch(key, *kIndexTermKeyRE, &token_id, &sort_key_, &doc_id_)) { + LOG("index: unable to parse token key: %s", key); + error_ = true; + return; + } + DCHECK_EQ(token_id, token_id_); +} + +void TokenResultIterator::GetRawTerms(StringSet* raw_terms) const { + if (!raw_prefix_.empty()) { + raw_terms->insert(raw_prefix_); + } +} + +} // namespace full_text_index + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/FullTextIndex.h b/clients/shared/FullTextIndex.h new file mode 100644 index 0000000..3155e71 --- /dev/null +++ b/clients/shared/FullTextIndex.h @@ -0,0 +1,319 @@ +// Copyright 2013 Viewfinder. All rights Reserved. +// Author: Ben Darnell + +#ifndef VIEWFINDER_FULL_TEXT_INDEX_H +#define VIEWFINDER_FULL_TEXT_INDEX_H + +#import +#import +#import "DB.h" +#import "Mutex.h" +#import "STLUtils.h" +#import "Utils.h" + +class AppState; +class FullTextIndex; + +class FullTextQuery { + public: + virtual ~FullTextQuery(); + + enum Type { + TERM, + AND, + OR, + PREFIX, + }; + + enum Options { + PREFIX_MATCH = 1 << 0, + }; + + virtual Type type() const = 0; + + virtual bool empty() const = 0; + + virtual string ToString() const = 0; + + static FullTextQuery* Parse(const Slice& query, int options=0); +}; + + +class FullTextQueryTermNode : public FullTextQuery { + public: + FullTextQueryTermNode(const Slice& term); + ~FullTextQueryTermNode(); + + Type type() const override { return TERM; } + + bool empty() const override { return false; } + + string ToString() const override { return term_; } + + const string& term() const { return term_; } + + private: + string term_; +}; + +class FullTextQueryPrefixNode : public FullTextQuery { + public: + FullTextQueryPrefixNode(const Slice& prefix); + ~FullTextQueryPrefixNode(); + + Type type() const override { return PREFIX; } + + bool empty() const override { return false; } + + string ToString() const override { return prefix_ + "*"; } + + const string& prefix() const { return prefix_; } + + private: + string prefix_; +}; + + +class FullTextQueryParentNode : public FullTextQuery { + public: + FullTextQueryParentNode(const vector& children); + ~FullTextQueryParentNode(); + + bool empty() const override { return children_.size() == 0; } + + string ToString() const override; + + const vector& children() const { return children_; } + + private: + vector children_; +}; + +class FullTextQueryAndNode : public FullTextQueryParentNode { + public: + FullTextQueryAndNode(const vector& children) + : FullTextQueryParentNode(children) { + } + Type type() const override { return AND; } +}; + +class FullTextQueryOrNode : public FullTextQueryParentNode { + public: + FullTextQueryOrNode(const vector& children) + : FullTextQueryParentNode(children) { + } + Type type() const override { return OR; } +}; + + +class FullTextQueryVisitor { + public: + virtual ~FullTextQueryVisitor(); + + void VisitNode(const FullTextQuery& node); + void VisitChildren(const FullTextQueryParentNode& node); + virtual void VisitTermNode(const FullTextQueryTermNode& node) { }; + virtual void VisitPrefixNode(const FullTextQueryPrefixNode& node) { }; + virtual void VisitParentNode(const FullTextQueryParentNode& node); + virtual void VisitAndNode(const FullTextQueryAndNode& node); + virtual void VisitOrNode(const FullTextQueryOrNode& node); +}; + + +class FullTextQueryTermExtractor : public FullTextQueryVisitor { + public: + // Adds all terms in the query to the given set. + explicit FullTextQueryTermExtractor(StringSet* terms) + : terms_(terms) { + } + + void VisitTermNode(const FullTextQueryTermNode& node) override { + terms_->insert(node.term()); + } + + void VisitPrefixNode(const FullTextQueryPrefixNode& node) override { + terms_->insert(node.prefix()); + } + + private: + StringSet* terms_; +}; + + +class FullTextResultIterator { + public: + virtual ~FullTextResultIterator(); + + // Returns true if the iterator is positioned at a valid result. + // Unless otherwise noted, other methods are undefined if Valid() returns false. + virtual bool Valid() const = 0; + + // Advances the iterator to the next result, or sets Valid() to false if there are no more results. + virtual void Next() = 0; + + // Advances to the first position >= other. Only moves forward, never backward, so if the current + // position is already >= other, nothing is changed. + virtual void Seek(const FullTextResultIterator& other); + + // Return the doc id for the current result. + virtual const Slice doc_id() const = 0; + + // Return the sort key for the current result. + virtual const Slice sort_key() const = 0; + + // If this result is derived from a normalized token, add the original denormalized form(s) to *raw_terms. + virtual void GetRawTerms(StringSet* raw_terms) const { + } + + protected: + FullTextResultIterator() {}; + + private: + // Disallow evil constructors. + FullTextResultIterator(const FullTextResultIterator&); + void operator=(const FullTextResultIterator&); +}; + + +class FullTextQueryIteratorBuilder : FullTextQueryVisitor { + public: + FullTextQueryIteratorBuilder(std::initializer_list indexes, const DBHandle& db); + ~FullTextQueryIteratorBuilder(); + + FullTextResultIterator* BuildIterator(const FullTextQuery& query); + + virtual void VisitTermNode(const FullTextQueryTermNode& node) override; + virtual void VisitPrefixNode(const FullTextQueryPrefixNode& node) override; + virtual void VisitParentNode(const FullTextQueryParentNode& node) override; + + private: + std::initializer_list indexes_; + const DBHandle& db_; + + typedef vector Accumulator; + vector stack_; +}; + +struct FullTextIndexTerm { + string index_term; + string raw_term; + int index; + static const char* kNameKeyFormat; + + FullTextIndexTerm(); + FullTextIndexTerm(const string& it, const string& rt, int i); + ~FullTextIndexTerm(); + string GetIndexTermKey(const Slice& prefix, const Slice& doc_id) const; +}; + +class FullTextIndex { + friend class FullTextQueryIteratorBuilder; + + public: + FullTextIndex(AppState* state, const Slice& name); + ~FullTextIndex(); + + // Returns a result iterator, which the caller is responsible for deleting. + // Typical usage: + // for (ScopedPtr iter(index->Search(db, query)); + // iter->Valid(); + // iter->Next()) { + // LoadDocument(iter->doc_id()); + // } + FullTextResultIterator* Search(const DBHandle& db, const FullTextQuery& query) const; + + // Retrieves a list of (frequency, string) pairs beginning with prefix. + // The results are sorted by descending frequency. + typedef vector > SuggestionResults; + void GetSuggestions(const DBHandle& db, const Slice& prefix, SuggestionResults* results); + + // Tokenizes 'phrase' and appends the resulting tokens to 'terms'. + // Multiple tokens may be generated for the same word based on + // punctuation removal and ascii transliteration. + int ParseIndexTerms(int index, const Slice& phrase, + vector* terms) const; + + // Adds 'token' to 'terms' without additional processing. + int AddVerbatimToken(int index, const Slice& token, + vector* terms) const; + + // Writes the given 'terms' to the database. 'disk_terms' is updated to contain the terms + // that were written, and should be persisted and passed back on future calls to UpdateIndex + // for this entity. + // 'sort_key' is used as a heuristic when there are too many results to retrieve; + // records with the lowest sort_keys are returned first. It may not contain spaces. + // All terms indexed for a given doc_id must have the same sort_key. + void UpdateIndex(const vector& terms, + const Slice& doc_id, const Slice& sort_key, + google::protobuf::RepeatedPtrField* disk_terms, + const DBHandle& updates); + + // Removes the given previously-indexed terms from the index, to be used when deleting a document. + void RemoveTerms(google::protobuf::RepeatedPtrField* disk_terms, + const DBHandle& updates); + + // Returns a string that will order results by decreasing timestamp. + static string TimestampSortKey(WallTime time); + + // Find and return a prefix of 'raw_term' corresponding to 'index_prefix'. 'index_prefix' is a prefix of + // some transliteration of 'raw_term'. For example, FindRawPrefix("vlad", "Владимир") returns + // "Влад". FindRawPefix Returns an empty string if a corresponding prefix could not be computed. Only public + // for tests. + static string FindRawPrefix(const Slice& index_prefix, const Slice& raw_term); + + // Remove all punctuation (actually all non-alphanumerics) from the given string. + // Only public for tests. + static string RemovePunctuation(const Slice& term); + + // Blocks until all pending background operations have completed. + // Only public for tests. + void DrainBackgroundOps(); + + // Returns the database prefix for the given token. + // For use in TokenResultIterator. + string FormatIndexTermPrefix(int64_t token_id) const; + + // Builds a filter regex that matches portions of a string that contributed to a search query. + // *all_terms must have been populated by calling GetRawTerms on a result iterator. + // and FullTextQueryTermExtractor on a parsed query. + // Caller takes ownership of the returned pointer. + static RE2* BuildFilterRE(const StringSet& all_terms); + + private: + // Generates additional index terms by lossily converting to 7-bit + // ascii. Adds index term and any denormalized version(s) of it to + // 'terms'. + void DenormalizeIndexTerm(int index, const string& term, + vector* terms) const; + + + string FormatLexiconKey(const Slice& term, const Slice& raw_term) const; + int64_t AddToLexicon(const FullTextIndexTerm& term); + void InvalidateLexiconStats(int64_t token_id, const DBHandle& updates); + + int64_t AllocateTokenIdLocked(const Slice& lex_key, const DBHandle& updates); + + void MaybeUpdateLexiconStats(); + // Returns true if any stats needed updating. + bool UpdateLexiconStats(); + + // Caller is responsible for deleting the iterator. + FullTextResultIterator* CreateTokenIterator(const DBHandle& db, const Slice& token) const; + FullTextResultIterator* CreateTokenPrefixIterator(const DBHandle& db, const Slice& token_prefix) const; + + AppState* state_; + const string name_; + const string index_prefix_; + const string lexicon_prefix_; + + std::unordered_map lexicon_cache_; + + // Held during AddToLexicon. + Mutex lexicon_mutex_; + + // Protects updating_lexicon_stats_. + Mutex lexicon_stats_mutex_; + bool updating_lexicon_stats_; +}; + +#endif // VIEWFINDER_FULL_TEXT_INDEX_H diff --git a/clients/shared/FullTextIndexInternal.h b/clients/shared/FullTextIndexInternal.h new file mode 100644 index 0000000..9a4b64b --- /dev/null +++ b/clients/shared/FullTextIndexInternal.h @@ -0,0 +1,102 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell +// +// This file contains implementation details of FullTextIndex which are only exposed for testing. + +#import "FullTextIndex.h" + +namespace full_text_index { + +class NullResultIterator : public FullTextResultIterator { + public: + NullResultIterator(); + ~NullResultIterator(); + + bool Valid() const override { + return false; + } + + void Next() override { }; + const Slice doc_id() const override { + return ""; + } + const Slice sort_key() const override { + return ""; + } +}; + +// Iterates over the intersection of a list of child iterators. +class AndResultIterator : public FullTextResultIterator { + public: + // Takes ownership of child iterators. + static FullTextResultIterator* Create(const vector& iterators); + ~AndResultIterator(); + + virtual bool Valid() const; + virtual void Next(); + + virtual const Slice doc_id() const; + virtual const Slice sort_key() const; + + virtual void GetRawTerms(StringSet* raw_terms) const; + + private: + explicit AndResultIterator(const vector& iterators); + + void SynchronizeIterators(); + + bool valid_; + vector iterators_; +}; + +// Iterates over the union of a list of child iterators. +class OrResultIterator : public FullTextResultIterator { + public: + // Takes ownership of child iterators. + static FullTextResultIterator* Create(const vector& iterators); + ~OrResultIterator(); + + virtual bool Valid() const; + virtual void Next(); + + virtual const Slice doc_id() const; + virtual const Slice sort_key() const; + + virtual void GetRawTerms(StringSet* raw_terms) const; + + private: + explicit OrResultIterator(const vector& iterators); + + vector iterators_; +}; + +// Iterates over all occurrences of a token. +class TokenResultIterator : public FullTextResultIterator { + public: + TokenResultIterator(const FullTextIndex& index, const DBHandle& db, int64_t token_id, const Slice& raw_term); + ~TokenResultIterator(); + + virtual bool Valid() const; + virtual void Next(); + + virtual const Slice doc_id() const { + return doc_id_; + } + virtual const Slice sort_key() const { + return sort_key_; + } + + virtual void GetRawTerms(StringSet* raw_terms) const; + + private: + void ParseHit(); + + const int64_t token_id_; + const string raw_prefix_; + DB::PrefixIterator db_iter_; + bool error_; + string doc_id_; + string sort_key_; +}; + +} // namespace full_text_index diff --git a/clients/shared/FullTextIndexMetadata.proto b/clients/shared/FullTextIndexMetadata.proto new file mode 100644 index 0000000..a1d4418 --- /dev/null +++ b/clients/shared/FullTextIndexMetadata.proto @@ -0,0 +1,12 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "FullTextIndexMetadataPB"; + +message FullTextLexiconMetadata { + optional int64 token_id = 1; + + // The number of times this token appears in the index. + optional int64 count = 2; +} diff --git a/clients/shared/GeocodeManager.h b/clients/shared/GeocodeManager.h new file mode 100644 index 0000000..0089c26 --- /dev/null +++ b/clients/shared/GeocodeManager.h @@ -0,0 +1,27 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_GEOCODE_MANAGER_H +#define VIEWFINDER_GEOCODE_MANAGER_H + +#import "Callback.h" + +class Location; +class Placemark; + +class GeocodeManager { + protected: + // Called when the reverse geocoding is complete. Placemark is NULL if + // geocoding failed. + typedef Callback Completion; + + public: + virtual ~GeocodeManager(); + + // Returns true if the location is already queued for reverse geocoding. + virtual bool ReverseGeocode(const Location* l, Completion completion) = 0; +}; + +GeocodeManager* NewGeocodeManager(); + +#endif // VIEWFINDER_GEOCODE_MANAGER_H diff --git a/clients/shared/GeocodeManager.ios.mm b/clients/shared/GeocodeManager.ios.mm new file mode 100644 index 0000000..5a5bfec --- /dev/null +++ b/clients/shared/GeocodeManager.ios.mm @@ -0,0 +1,139 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import "GeocodeManager.h" +#import "Location.pb.h" +#import "Logging.h" +#import "Placemark.pb.h" +#import "STLUtils.h" + +namespace { + +class GeocodeManagerIOS : public GeocodeManager { + struct Pending { + const Location* l; + Completion completion; + }; + + public: + GeocodeManagerIOS() + : active_(false), + geocoder_([CLGeocoder new]) { + } + + ~GeocodeManagerIOS() { + } + + // Returns true if the location is already queued for reverse geocoding. + virtual bool ReverseGeocode(const Location* l, Completion completion) { + if (ContainsKey(locations_, l)) { + return false; + } + locations_.insert(l); + queue_.push_back(Pending()); + queue_.back().l = l; + queue_.back().completion = completion; + MaybeProcessQueue(); + return true; + } + + private: + void MaybeProcessQueue() { + if (active_ || queue_.empty()) { + return; + } + active_ = true; + + Pending p = queue_.front(); + CLLocation* l = [[CLLocation alloc] + initWithLatitude:p.l->latitude() + longitude:p.l->longitude()]; + VLOG("geocode: start: %f,%f", l.coordinate.latitude, l.coordinate.longitude); + + [geocoder_ + reverseGeocodeLocation:l + completionHandler:^(NSArray* placemarks, NSError* error) { + VLOG("geocode: finish: %f,%f: %s", + l.coordinate.latitude, l.coordinate.longitude, placemarks); + Placemark storage; + Placemark* m = NULL; + + if (error) { + if (error.domain == kCLErrorDomain && + (error.code == kCLErrorGeocodeFoundPartialResult || + error.code == kCLErrorNetwork)) { + // We're being throttled. Retry after a delay. + LOG("geocode: throttled"); + dispatch_after_main(5, [this] { + active_ = false; + MaybeProcessQueue(); + }); + return; + } else { + // A more permanent error. Skip reverse geocoding this location. + LOG("geocode: error: %@", error); + } + } else { + for (int i = 0; i < placemarks.count; ++i) { + CLPlacemark* mark = [placemarks objectAtIndex:i]; + NSDictionary* address = mark.addressDictionary; + m = &storage; + if (mark.ISOcountryCode) { + m->set_iso_country_code(ToString(mark.ISOcountryCode)); + } + if (mark.country) { + m->set_country(ToString(mark.country)); + } + NSString* state = address[@"State"]; + if (state) { + m->set_state(ToString(state)); + } + if (mark.postalCode) { + m->set_postal_code(ToString(mark.postalCode)); + } + if (mark.locality) { + m->set_locality(ToString(mark.locality)); + } + if (mark.subLocality) { + m->set_sublocality(ToString(mark.subLocality)); + } + if (mark.thoroughfare) { + m->set_thoroughfare(ToString(mark.thoroughfare)); + } + if (mark.subThoroughfare) { + m->set_subthoroughfare(ToString(mark.subThoroughfare)); + } + break; + } + } + + active_ = false; + queue_.pop_front(); + locations_.erase(p.l); + p.completion(m); + MaybeProcessQueue(); + }]; + } + + private: + bool active_; + CLGeocoder* geocoder_; + std::deque queue_; + std::unordered_set locations_; +}; + +} // namespace + +GeocodeManager::~GeocodeManager() { +} + +GeocodeManager* NewGeocodeManager() { + return new GeocodeManagerIOS; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/IdentityManager.cc b/clients/shared/IdentityManager.cc new file mode 100644 index 0000000..99d0dc1 --- /dev/null +++ b/clients/shared/IdentityManager.cc @@ -0,0 +1,122 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import +#import +#import "IdentityManager.h" +#import "LazyStaticPtr.h" +#import "LocaleUtils.h" +#import "Logging.h" + +// Authority types. +const string IdentityManager::kViewfinderAuthority = "Viewfinder"; +const string IdentityManager::kGoogleAuthority = "Google"; +const string IdentityManager::kFacebookAuthority = "Facebook"; + +// Identity types (by prefix). +const string IdentityManager::kEmailIdentityPrefix = "Email:"; +const string IdentityManager::kPhoneIdentityPrefix = "Phone:"; +const string IdentityManager::kFacebookIdentityPrefix = "FacebookGraph:"; +const string IdentityManager::kViewfinderIdentityPrefix = "VF:"; + +namespace { + +// Quick check for phone numbers in E164 format: starts with a plus, and then only digits. +LazyStaticPtr kPhoneNumberRE = { "^\\+[0-9]+$" }; + +} // namespace + +string IdentityManager::IdentityToName(const Slice& identity) { + if (IsEmailIdentity(identity)) { + return EmailFromIdentity(identity); + } else if (IsPhoneIdentity(identity)) { + return PhoneFromIdentity(identity); + } + return string(); +} + +string IdentityManager::IdentityToDisplayName(const Slice& identity) { + if (IsEmailIdentity(identity)) { + return EmailFromIdentity(identity); + } else if (IsPhoneIdentity(identity)) { + return PhoneFromIdentity(identity); + } else if (IsFacebookIdentity(identity)) { + return "Facebook"; + } + return string(); +} + +string IdentityManager::IdentityType(const Slice& identity) { + if (IsEmailIdentity(identity)) { + return "email"; + } else if (IsPhoneIdentity(identity)) { + return "mobile"; + } else if (IsFacebookIdentity(identity)) { + return "Facebook"; + } else if (IsViewfinderIdentity(identity)) { + return "Viewfinder"; + } + return string(); +} + +bool IdentityManager::IsEmailIdentity(const Slice& identity) { + return identity.starts_with(kEmailIdentityPrefix); +} + +bool IdentityManager::IsPhoneIdentity(const Slice& identity) { + return identity.starts_with(kPhoneIdentityPrefix); +} + +bool IdentityManager::IsFacebookIdentity(const Slice& identity) { + return identity.starts_with(kFacebookIdentityPrefix); +} + +bool IdentityManager::IsViewfinderIdentity(const Slice& identity) { + return identity.starts_with(kViewfinderIdentityPrefix); +} + +string IdentityManager::IdentityForUserId(int64_t user_id) { + return Format("%s%s", kViewfinderIdentityPrefix, user_id); +} + +string IdentityManager::IdentityForEmail(const string& email) { + return Format("%s%s", kEmailIdentityPrefix, ToLowercase(email)); +} + +string IdentityManager::IdentityForPhone(const Slice& phone) { + CHECK(RE2::FullMatch(phone, *kPhoneNumberRE)); + return Format("%s%s", kPhoneIdentityPrefix, phone); +} + +string IdentityManager::EmailFromIdentity(const Slice& identity) { + if (IsEmailIdentity(identity)) { + return identity.substr(kEmailIdentityPrefix.size()).ToString(); + } + return string(); +} + +string IdentityManager::PhoneFromIdentity(const Slice& identity) { + const string raw_phone = RawPhoneFromIdentity(identity); + using i18n::phonenumbers::PhoneNumberUtil; + i18n::phonenumbers::PhoneNumber number; + PhoneNumberUtil* phone_util = PhoneNumberUtil::GetInstance(); + PhoneNumberUtil::ErrorType error = phone_util->Parse(raw_phone, "ZZ", &number); + if (error != PhoneNumberUtil::NO_PARSING_ERROR) { + return string(); + } + const bool in_country = (number.country_code() == phone_util->GetCountryCodeForRegion(GetPhoneNumberCountryCode())); + string formatted; + phone_util->Format(number, in_country ? PhoneNumberUtil::NATIONAL : PhoneNumberUtil::INTERNATIONAL, &formatted); + return formatted; +} + +string IdentityManager::RawPhoneFromIdentity(const Slice& identity) { + if (!IsPhoneIdentity(identity)) { + return string(); + } + return ToString(identity.substr(kPhoneIdentityPrefix.size())); +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/IdentityManager.h b/clients/shared/IdentityManager.h new file mode 100644 index 0000000..5d0cfee --- /dev/null +++ b/clients/shared/IdentityManager.h @@ -0,0 +1,68 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_IDENTITY_MANAGER_H +#define VIEWFINDER_IDENTITY_MANAGER_H + +#import "Utils.h" + +// The IdentityManager class handles identity syntax and parsing. +class IdentityManager { + public: + // Returns a human-readable representation of a contact identity for + // contact indexing. + static string IdentityToName(const Slice& identity); + + // Returns a human-readable representation of a contact identity for + // display in the UI. + static string IdentityToDisplayName(const Slice& identity); + + // Returns the type of identity in a human-readable representation. + static string IdentityType(const Slice& identity); + + static bool IsEmailIdentity(const Slice& identity); + static bool IsPhoneIdentity(const Slice& identity); + static bool IsFacebookIdentity(const Slice& identity); + static bool IsViewfinderIdentity(const Slice& identity); + + // Returns the identity string for the given viewfinder user id. + // Note that these are "fake" identities not supported by the server. + static string IdentityForUserId(int64_t user_id); + + // Returns the identity string for the given email address. + // Converts the address to a canonical form, consistent with the backend + // Identity.CanonicalizeEmail() function. + static string IdentityForEmail(const string& email); + + // Phone number must be in normalized (E164) format. + static string IdentityForPhone(const Slice& phone); + + // Extracts the email address from the identity. Returns an empty string if + // no email address can be found. + static string EmailFromIdentity(const Slice& identity); + + // Extracts the phone number from the identity (formatted for the current locale. + // Returns an empty string if it is not a phone identity. + static string PhoneFromIdentity(const Slice& identity); + + // Returns the unformatted (E164) version of the phone number for this identity. + static string RawPhoneFromIdentity(const Slice& identity); + + private: + // Authority types. + static const string kViewfinderAuthority; + static const string kGoogleAuthority; + static const string kFacebookAuthority; + + // Identity types (by prefix). + static const string kEmailIdentityPrefix; + static const string kPhoneIdentityPrefix; + static const string kFacebookIdentityPrefix; + static const string kViewfinderIdentityPrefix; +}; + +#endif // VIEWFINDER_IDENTITY_MANAGER_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ImageFingerprint.proto b/clients/shared/ImageFingerprint.proto new file mode 100644 index 0000000..c5a38c6 --- /dev/null +++ b/clients/shared/ImageFingerprint.proto @@ -0,0 +1,13 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ImageFingerprintPB"; + +message ImageFingerprint { + // An image fingerprint is composed of a number of terms: 160-bit binary + // strings. Each term is indexed so that a match on any term is considered a + // match of the images. There might be multiple terms for an image if we were + // unable to robustly normalize the image orientation. + repeated bytes terms = 1; +} diff --git a/clients/shared/ImageFingerprintPrivate.h b/clients/shared/ImageFingerprintPrivate.h new file mode 100644 index 0000000..d9ee87b --- /dev/null +++ b/clients/shared/ImageFingerprintPrivate.h @@ -0,0 +1,24 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis +// +// Constants shared by the ImageFingerprint and ImageIndex code. + +#ifndef VIEWFINDER_IMAGE_FINGERPRINT_PRIVATE_H +#define VIEWFINDER_IMAGE_FINGERPRINT_PRIVATE_H + +#include + +const int kHaarSmallN = 32; +const int kHaarSmallNxN = kHaarSmallN * kHaarSmallN; +const int kHaarHashN = 13; +const int kHaarHashNxN = kHaarHashN * kHaarHashN; +const int kHaarHashBits = 160; +const int kHaarHashBytes = (kHaarHashBits + 7) / 8; +// Skip the first entry of haar_data (in the zig-zag traversal) because it +// gives little information. The first entry is the average pixel value across +// the image. +const int kHaarHashSkip = 1; + +extern const std::vector kZigZagOffsets; + +#endif // VIEWFINDER_IMAGE_FINGERPRINT_PRIVATE_H diff --git a/clients/shared/ImageIndex.cc b/clients/shared/ImageIndex.cc new file mode 100644 index 0000000..8f7a02d --- /dev/null +++ b/clients/shared/ImageIndex.cc @@ -0,0 +1,385 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// +// The idea behind the perceptual fingerprint is based on the papers "Fast +// Multiresolution Image Querying" - Jacobs, et al and "Image Similarity Search +// with Compact Data Structures" - Lv, et al. The high-level: downsize the +// image to 32x32 grayscale (i.e. only consider luminance). The downsizing +// removes noise and some minor compression artifacts. Apply a 5x5 box blur to +// remove more noise/compression artifacts. Normalize the image +// orientation. Apply the discrete Haar wavelet transform which computes +// horizontal and vertical image gradients at various scales. Create a +// fingerprint using the resulting gradients by setting a bit in the +// fingerprint if the corresponding gradient has a non-negative value. The +// resulting fingerprint can be compared with another fingerprint using a +// simple hamming distance calculation. Similar fingerprints will correspond to +// similar images. Why does this work? The fingerprint captures image gradients +// at multiple scale levels and similar images will have similar gradient +// profiles. +// +// Note that we can't use an exact match on the fingerprint for +// searching. Minor compression artifacts will still cause a few bits in the +// fingerprint to change. When searching for a matching fingerprint, brute +// force would be possible. The hamming distance calculation is extremely fast +// (xor + number-of-bits-set). But we can do a bit better since we're only +// searching for near-duplicate images with a small number of differences from +// a target fingerprint. +// +// The perceptual hash contains 160 bits. When performing a search, we want to +// find the other fingerprints which have a small hamming distance from our +// search fingerprint. For our fingerprint, a hamming distance <= 5% of the +// fingerprint length usually gives solid confirmation of near-duplicate +// images. 160 bits * 5% == 8 bits. +// +// When adding a fingerprint to the index, we index 12-bit N-grams (tags) for +// each 160-bit fingerprint. There are 13 non-overlapping 12-bit N-grams in +// each fingerprint and, by the pigeon-hole principle, we are guaranteed that 2 +// fingerprints with a hamming distance <= 12 will contain at least 1 N-gram in +// common. +// +// A 12-bit tag size provides 4096 buckets. With 100,000 unique images indexed, +// we would expect ~25 fingerprints per bucket. A search needs to examine all +// 13 buckets for a fingerprint giving an expected 325 fingerprints comparisons +// per search. +// +// TODO(peter): Can/should we incorporate chrominance into the fingerprint? + +#import "DBFormat.h" +#import "ImageFingerprintPrivate.h" +#import "ImageIndex.h" +#import "Logging.h" +#import "Timer.h" + +namespace { + +// The tag lengths are expressed in hexadecimal characters (i.e. 4 bits) and +// must sum to 40 characters. +const int kTagLengths[13] = { + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, +}; +const int kMatchThreshold = 12; + +const string kImageIndexKeyPrefix = DBFormat::image_index_key(""); + +const DBRegisterKeyIntrospect kImageIndexKeyIntrospect( + kImageIndexKeyPrefix, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +// #define SQUARE_ZIG_ZAG +#ifdef SQUARE_ZIG_ZAG + +// Generate a vector of offsets for the zig-zag traversal of the upper-left +// NxN square region of an MxM matrix. +vector MakeSquareZigZagOffsets(int n, int m) { + assert(m >= n); + vector offsets(n * n); + int i = 0, j = 0; + int d = -1; + int start = 0; + int end = n * n - 1; + do { + offsets[start++] = i * m + j; + offsets[end--] = (n - i - 1) * m + n - j - 1; + i += d; + j -= d; + if (i < 0) { + ++i; + d = -d; + } else if (j < 0) { + ++j; + d = -d; + } + } while (start < end); + if (start == end) { + offsets[start] = i * m + j; + } + return offsets; +} + +#else // !SQUARE_ZIG_ZAG + +// Generate a vector of offsets for the zig-zag traversal of the first K cells of an NxN matrix. +vector MakeTriangularZigZagOffsets(int k, int n) { + assert(k < (n * n)); + vector offsets(k); + int i = 0, j = 0; + int d = -1; + int start = 0; + int end = n * n - 1; + do { + offsets[start++] = i * n + j; + if (end < k) { + offsets[end] = (n - i) * n - j - 1; + } + end--; + i += d; + j -= d; + if (i < 0) { + ++i; + d = -d; + } else if (j < 0) { + ++j; + d = -d; + } + } while (start < k && start < end); + if (start == end && start < k) { + offsets[start] = i * n + j; + } + return offsets; +} + +#endif // !SQUARE_ZIG_ZAG + +string EncodeKey(const Slice& hex_term, int i, int n, const string& id) { + return Format("%s%02d:%s#%s", kImageIndexKeyPrefix, + i, hex_term.substr(i, n), id); +} + +bool DecodeKey(Slice key, Slice* tag, Slice* id) { + if (!key.starts_with(kImageIndexKeyPrefix)) { + return false; + } + key.remove_prefix(kImageIndexKeyPrefix.size()); + const int pos = key.find('#'); + if (pos == key.npos) { + return false; + } + if (tag) { + *tag = key.substr(0, pos); + } + if (id) { + *id = key.substr(pos + 1); + } + return true; +} + +StringSet GenerateKeys(const ImageFingerprint& f, const string& id) { + StringSet keys; + for (int i = 0; i < f.terms_size(); ++i) { + const string hex_term = BinaryToHex(f.terms(i)); + const int n = hex_term.size(); + for (int j = 0, k = 0, len; j < n; j += len, ++k) { + len = kTagLengths[k]; + keys.insert(EncodeKey(hex_term, j, len, id)); + } + } + return keys; +} + +string Intersect(const Slice& a, const Slice& b) { + DCHECK_EQ(a.size(), b.size()); + string r(a.ToString()); + for (int i = 0; i < r.size(); ++i) { + r[i] ^= b[i]; + } + return r; +} + +int HammingDistance(const Slice& a, const Slice& b) { + DCHECK_EQ(a.size(), b.size()); + DCHECK_EQ(0, a.size() % 4); + int count = 0; + const uint32_t* a_ptr = (const uint32_t*)a.data(); + const uint32_t* b_ptr = (const uint32_t*)b.data(); + for (int i = 0, n = a.size() / 4; i < n; ++i) { + count += __builtin_popcount(a_ptr[i] ^ b_ptr[i]); + } + return count; +} + +} // namespace + +#ifdef SQUARE_ZIG_ZAG +const vector kZigZagOffsets = MakeSquareZigZagOffsets( + kHaarHashN, kHaarSmallN); +#else // !SQUARE_ZIG_ZAG +const vector kZigZagOffsets = MakeTriangularZigZagOffsets( + kHaarHashBits + kHaarHashSkip, kHaarSmallN); +#endif // !SQUARE_ZIG_ZAG + +ostream& operator<<(ostream& os, const ImageFingerprint& f) { + for (int i = 0; i < f.terms_size(); ++i) { + if (i > 0) { + os << ":"; + } + os << BinaryToHex(f.terms(i)); + } + return os; +} + +ImageIndex::ImageIndex(bool histogram) + : histogram_(histogram ? new vector(kHaarHashBits) : NULL), + histogram_count_(0) { +} + +ImageIndex::~ImageIndex() { +} + +void ImageIndex::Add(const ImageFingerprint& fingerprint, + const string& id, const DBHandle& updates) { + const StringSet keys = GenerateKeys(fingerprint, id); + for (StringSet::const_iterator iter(keys.begin()); + iter != keys.end(); + ++iter) { + updates->PutProto(*iter, fingerprint); + } + + if (histogram_.get()) { + // Keep a histogram of the bits set in the fingerprints so that we can + // verify in tests that every bit is being used. + for (int i = 0; i < fingerprint.terms_size(); ++i) { + const string& s = fingerprint.terms(i); + ++histogram_count_; + const uint8_t* p = (const uint8_t*)s.data(); + for (int j = 0; j < s.size() * 8; ++j) { + if (p[j / 8] & (1 << (j % 8))) { + (*histogram_)[j] += 1; + } + } + } + } +} + +void ImageIndex::Remove(const ImageFingerprint& fingerprint, + const string& id, const DBHandle& updates) { + const StringSet keys = GenerateKeys(fingerprint, id); + for (StringSet::const_iterator iter(keys.begin()); + iter != keys.end(); + ++iter) { + updates->Delete(*iter); + } +} + +int ImageIndex::Search(const DBHandle& db, const ImageFingerprint& fingerprint, + StringSet* matched_ids) const { + const StringSet keys = GenerateKeys(fingerprint, ""); + std::unordered_set checked_ids; + int candidates = 0; + for (StringSet::const_iterator key_iter(keys.begin()); + key_iter != keys.end(); + ++key_iter) { + for (DB::PrefixIterator iter(db, *key_iter); + iter.Valid(); + iter.Next()) { + Slice id; + if (!DecodeKey(iter.key(), NULL, &id)) { + DCHECK(false); + continue; + } + const string id_str = id.ToString(); + if (ContainsKey(checked_ids, id_str)) { + continue; + } + ++candidates; + ImageFingerprint candidate_fingerprint; + if (!candidate_fingerprint.ParseFromArray( + iter.value().data(), iter.value().size())) { + DCHECK(false); + continue; + } + checked_ids.insert(id_str); + const int n = HammingDistance(fingerprint, candidate_fingerprint); + if (n > kMatchThreshold) { + continue; + } + matched_ids->insert(id_str); + } + } + return candidates; +} + +string ImageIndex::PrettyHistogram() const { + if (!histogram_.get()) { + return ""; + } + vector v(kHaarSmallN * kHaarSmallN); + if (histogram_count_ > 0) { + for (int i = 0; i < kHaarHashSkip; ++i) { + v[kZigZagOffsets[i]] = " "; + } + for (int i = 0; i < histogram_->size(); ++i) { + const int val = (100 * (*histogram_)[i]) / histogram_count_; + v[kZigZagOffsets[i + kHaarHashSkip]] = Format("%3d", val); + } + } + string s; + for (int i = 0; i < kHaarSmallN; ++i) { + if (v[i + kHaarSmallN].empty()) { + break; + } + for (int j = 0; j < kHaarSmallN; ++j) { + const string& t = v[i * kHaarSmallN + j]; + if (t.empty()) { + break; + } + s += " " + t; + } + s += "\n"; + } + return s; +} + +int ImageIndex::TotalTags(const DBHandle& db) const { + int count = 0; + for (DB::PrefixIterator iter(db, kImageIndexKeyPrefix); + iter.Valid(); + iter.Next()) { + ++count; + } + return count; +} + +int ImageIndex::UniqueTags(const DBHandle& db) const { + string last_tag; + int count = 0; + for (DB::PrefixIterator iter(db, kImageIndexKeyPrefix); + iter.Valid(); + iter.Next()) { + Slice tag; + if (!DecodeKey(iter.key(), &tag, NULL)) { + DCHECK(false); + continue; + } + if (last_tag != tag) { + ++count; + last_tag = tag.ToString(); + } + } + return count; +} + +ImageFingerprint ImageIndex::Intersect( + const ImageFingerprint& a, const ImageFingerprint& b) { + ImageFingerprint f; + int best_dist = std::numeric_limits::max(); + int best_index_a = -1; + int best_index_b = -1; + for (int i = 0; i < a.terms_size(); ++i) { + for (int j = 0; j < b.terms_size(); ++j) { + const int dist = ::HammingDistance(a.terms(i), b.terms(j)); + if (best_dist > dist) { + best_dist = dist; + best_index_a = i; + best_index_b = j; + } + } + } + f.add_terms(::Intersect(a.terms(best_index_a), b.terms(best_index_b))); + return f; +} + +int ImageIndex::HammingDistance( + const ImageFingerprint& a, const ImageFingerprint& b) { + int best = std::numeric_limits::max(); + for (int i = 0; i < a.terms_size(); ++i) { + for (int j = 0; j < b.terms_size(); ++j) { + best = std::min(best, ::HammingDistance(a.terms(i), b.terms(j))); + } + } + return best; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ImageIndex.h b/clients/shared/ImageIndex.h new file mode 100644 index 0000000..87839b5 --- /dev/null +++ b/clients/shared/ImageIndex.h @@ -0,0 +1,54 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_IMAGE_INDEX_H +#define VIEWFINDER_IMAGE_INDEX_H + +#import +#import +#import "DB.h" +#import "ImageFingerprint.pb.h" +#import "StringUtils.h" +#import "Utils.h" + +class ImageIndex { + public: + ImageIndex(bool histogram = false); + ~ImageIndex(); + + // Adds the specified fingerprint and id to the index. + void Add(const ImageFingerprint& fingerprint, + const string& id, const DBHandle& updates); + + // Removes the specified fingerprint and id from the index. + void Remove(const ImageFingerprint& fingerprint, + const string& id, const DBHandle& updates); + + // Search the index for the specified fingerprint, returning the ids of any + // matching fingerprints. + int Search(const DBHandle& db, const ImageFingerprint& fingerprint, + StringSet* matched_ids) const; + + // Returns a "pretty" histogram of the bits that were set in indexed + // fingerprints. Note that this histogram is not persistent and is only + // mainted if "true" was passed to the histogram parameter of the + // constructor. + string PrettyHistogram() const; + + // Return the total/unique indexed tags. These functions are slow. + int TotalTags(const DBHandle& db) const; + int UniqueTags(const DBHandle& db) const; + + static ImageFingerprint Intersect( + const ImageFingerprint& a, const ImageFingerprint& b); + static int HammingDistance( + const ImageFingerprint& a, const ImageFingerprint& b); + + private: + ScopedPtr > histogram_; + int histogram_count_; +}; + +ostream& operator<<(ostream& os, const ImageFingerprint& f); + +#endif // VIEWFINDER_IMAGE_INDEX_H diff --git a/clients/shared/InvalidateMetadata.proto b/clients/shared/InvalidateMetadata.proto new file mode 100644 index 0000000..d211a23 --- /dev/null +++ b/clients/shared/InvalidateMetadata.proto @@ -0,0 +1,69 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "InvalidateMetadataPB"; + +message ContactSelection { + optional string start_key = 1; + + // If set, all existing contacts should be erased and re-queried. + // Should not be saved to the database; it is only used for + // communication between the notification manager and contact + // manager. + optional bool all = 2; +} + +message EpisodeSelection { + optional string episode_id = 1; + optional bool get_attributes = 2 [default=false]; + optional bool get_photos = 3 [default=false]; + optional string photo_start_key = 4; + // IMPORTANT: update the handler in EpisodeTable.mm to reflect + // any additional invalidation attributes as they are added. +} + +message ViewpointSelection { + optional string viewpoint_id = 1; + optional bool get_attributes = 2 [default=false]; + optional bool get_followers = 3 [default=false]; + optional string follower_start_key = 4; + optional bool get_activities = 5 [default=false]; + optional string activity_start_key = 6; + optional bool get_episodes = 7 [default=false]; + optional string episode_start_key = 8; + optional bool get_comments = 9 [default=false]; + optional string comment_start_key = 10; + // IMPORTANT: update the handler in ViewpointTable.mm to reflect + // any additional invalidation attributes as they are added. +} + +message UserSelection { + optional int64 user_id = 1; +} + +message InvalidateMetadata { + optional bool all = 1; + + repeated ViewpointSelection viewpoints = 2; + repeated EpisodeSelection episodes = 3; + optional ContactSelection contacts = 4; + repeated UserSelection users = 5; +} + +message NotificationSelection { + // The last received notification key. + optional string last_key = 1; + // True if all existing notifications have been queried. + optional bool query_done = 2; + + // The minimum notification id received for a notification with a + // min_required_version indicating that the client is too old to + // fully understand invalidation. + optional int64 low_water_notification_id = 3; + + // The maximum of all min_required_version values received by the + // client. Once the client has reached this version, old + // notifications may be productively re-queried. + optional int32 max_min_required_version = 4; +} diff --git a/clients/shared/JsonUtils.cc b/clients/shared/JsonUtils.cc new file mode 100644 index 0000000..d8bce5c --- /dev/null +++ b/clients/shared/JsonUtils.cc @@ -0,0 +1,73 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "JsonUtils.h" +#import "Logging.h" +#import "StringUtils.h" + +string JsonRef::Format() const { +#ifdef DEBUG + return FormatStyled(); +#else // DEBUG + return FormatCompact(); +#endif // DEBUG +} + +string JsonRef::FormatStyled() const { + Json::StyledWriter writer; + return writer.write(const_value_); +} + +string JsonRef::FormatCompact() const { + Json::FastWriter writer; + return writer.write(const_value_); +} + +bool JsonRef::Contains(const char* key) const { + if (const_value_.type() != Json::objectValue) { + return false; + } + return const_value_.isMember(key); +} + +JsonRef JsonRef::operator[](int index) const { + if (const_value_.type() != Json::arrayValue) { + return Json::Value::null; + } + return const_value_[index]; +} + +JsonRef JsonRef::operator[](const char* key) const { + if (const_value_.type() != Json::objectValue) { + return Json::Value::null; + } + return const_value_[key]; +} + +string JsonRef::string_value() const { + if (const_value_.type() == Json::arrayValue || + const_value_.type() == Json::objectValue) { + return string(); + } + return const_value_.asString(); +} + +bool JsonValue::Parse(const string& data) { + if (data.empty()) { + return true; + } + Json::Reader reader; + if (!reader.parse(data, value_)) { + LOG("network: error parsing json: %s\n%s", + reader.getFormatedErrorMessages(), data); + value_ = Json::Value(); + return false; + } + return true; +} + +JsonValue ParseJSON(const string& data) { + JsonValue root; + root.Parse(data); + return JsonValue(root); +} diff --git a/clients/shared/JsonUtils.h b/clients/shared/JsonUtils.h new file mode 100644 index 0000000..3600955 --- /dev/null +++ b/clients/shared/JsonUtils.h @@ -0,0 +1,198 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_JSON_UTILS_H +#define VIEWFINDER_JSON_UTILS_H + +#import +#import +#import +#import "Utils.h" + +// A wrapper around Json::Value that allows us to define better semantics for +// various operations. Note that JsonRef holds a const-reference and only +// contains const methods. Mutable methods are part of JsonValue. +class JsonRef { + public: + JsonRef(const JsonRef& v) + : const_value_(v.const_value_) { + } + JsonRef(const Json::Value& v) + : const_value_(v) { + } + + // Returns the formatted json value. + string Format() const; + string FormatStyled() const; + string FormatCompact() const; + + // Returns true if the value is an object and contains the specified key. + bool Contains(const char* key) const; + + // Retrieves the value at index, returning Json::Value::null if the value is + // not an array or the index is invalid. + JsonRef operator[](int index) const; + + // Retrieves the value at key, return Json::Value::null if the value is non + // at object or the key is not present. + JsonRef operator[](const char* key) const; + + Json::Value::Members member_names() const { + if (const_value_.type() != Json::objectValue) { + return Json::Value::Members(); + } + return const_value_.getMemberNames(); + } + + bool empty() const { + return const_value_.empty(); + } + int size() const { + return const_value_.size(); + } + + bool bool_value() const { + return const_value_.asBool(); + } + int32_t int32_value() const { + return const_value_.asInt(); + } + int64_t int64_value() const { + return const_value_.asInt64(); + } + double double_value() const { + return const_value_.asDouble(); + } + string string_value() const; + + private: + const Json::Value& const_value_; +}; + +class JsonValue : public JsonRef { + friend class JsonArray; + friend class JsonDict; + + public: + JsonValue(const JsonValue& v) + : JsonRef(value_), + value_(v.value_) { + } + JsonValue(Json::ValueType type = Json::nullValue) + : JsonRef(value_), + value_(type) { + } + JsonValue(bool v) + : JsonRef(value_), + value_(v) { + } + JsonValue(int32_t v) + : JsonRef(value_), + value_(v) { + } + JsonValue(uint32_t v) + : JsonRef(value_), + value_(v) { + } + JsonValue(int64_t v) + : JsonRef(value_), + value_(v) { + } + JsonValue(uint64_t v) + : JsonRef(value_), + value_(v) { + } + JsonValue(double v) + : JsonRef(value_), + value_(v) { + } + JsonValue(const char* v) + : JsonRef(value_), + value_(v) { + } + JsonValue(const string& v) + : JsonRef(value_), + value_(v) { + } + JsonValue(const Json::Value& v) + : JsonRef(value_), + value_(v) { + } + // Array initializer. + JsonValue(std::initializer_list init) + : JsonRef(value_), + value_(Json::arrayValue) { + for (auto v : init) { + value_[size()] = v.value_; + } + } + // Object initializer. + JsonValue(std::initializer_list> init) + : JsonRef(value_), + value_(Json::objectValue) { + for (auto v : init) { + value_[v.first] = v.second.value_; + } + } + + // Parses the json data, returning true if the data could be parsed and false + // otherwise. + bool Parse(const string& data); + + private: +#ifdef __OBJC__ + // Do not allow a JsonValue to be constructed from an objective-c object. + JsonValue(id v); +#endif // __OBJC__ + + protected: + Json::Value value_; +}; + +class JsonArray : public JsonValue { + public: + JsonArray() + : JsonValue(Json::arrayValue) { + } + JsonArray(const JsonArray& a) + : JsonValue(a) { + } + JsonArray(std::initializer_list init) + : JsonValue(init) { + } + JsonArray(int count, const std::function& generator) + : JsonArray() { + for (int i = 0; i < count; ++i) { + push_back(generator(i)); + } + } + + void push_back(const JsonValue& v) { + value_[size()] = v.value_; + } +}; + +class JsonDict : public JsonValue { + public: + JsonDict() + : JsonValue(Json::objectValue) { + } + JsonDict(const JsonDict& d) + : JsonValue(d) { + } + JsonDict(const char* key, const JsonValue& value) + : JsonDict() { + insert(key, value); + } + JsonDict(std::initializer_list> init) + : JsonValue(init) { + } + + void insert(const char* key, const JsonValue& v) { + value_[key] = v.value_; + } +}; + +JsonValue ParseJSON(const string& data); + +#endif // VIEWFINDER_JSON_UTILS_H diff --git a/clients/shared/LazyStaticPtr.h b/clients/shared/LazyStaticPtr.h new file mode 100644 index 0000000..a3c5bc7 --- /dev/null +++ b/clients/shared/LazyStaticPtr.h @@ -0,0 +1,90 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_LAZY_STATIC_PTR_H +#define VIEWFINDER_LAZY_STATIC_PTR_H + +#import + +template +class LazyStaticPtr; + +namespace internal { + +struct NoArg { }; + +template +struct Helper { + static void Init(LazyStaticPtr* p) { + p->u_.ptr_ = new Type(p->arg1_, p->arg2_, p->arg3_); + } +}; + +template +struct Helper { + static void Init(LazyStaticPtr* p) { + p->u_.ptr_ = new Type(p->arg1_, p->arg2_); + } +}; + +template +struct Helper { + static void Init(LazyStaticPtr* p) { + p->u_.ptr_ = new Type(p->arg1_); + } +}; + +template +struct Helper { + static void Init(LazyStaticPtr* p) { + p->u_.ptr_ = new Type(); + } +}; + +} // namespace internal + +template +class LazyStaticPtr { + typedef internal::NoArg NoArg; + typedef internal::Helper Helper; + + public: + Type& operator*() { return *get(); } + Type* operator->() { return get(); } + + Type* get() { + typedef void (*function_t)(void*); + // Note(peter): This sucks. If std::once_flag is a member of LazyStaticPtr + // then the compiler generates a constructor for LazyStaticPtr which clears + // out the various members at an arbitrary time in the initialization + // process which is very problematic if we've already called + // LazyStaticPtr::get() for that LazyStaticPtr instance. We don't want + // LazyStaticPtr to have a constructor and instead want it to be statically + // initialized. There is likely some bit of c++11 magic that could make + // this hack go away, I just haven't found it yet. Or maybe this is a + // compiler bug. + std::once_flag* once = reinterpret_cast(once_buf_); + std::call_once(*once, &Helper::Init, this); + return u_.ptr_; + } + + public: + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + + union { + Type* ptr_; + } u_; + + char once_buf_[sizeof(std::once_flag)]; + + private: + // Disable assignment. + void operator=(const LazyStaticPtr&); +}; + +#endif // VIEWFINDER_LAZY_STATIC_PTR_H diff --git a/clients/shared/LocaleUtils.android.cc b/clients/shared/LocaleUtils.android.cc new file mode 100644 index 0000000..11f2aea --- /dev/null +++ b/clients/shared/LocaleUtils.android.cc @@ -0,0 +1,10 @@ +// Copryright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault + +#include "LocaleUtils.h" + +string phone_number_country_code = "US"; + +string GetPhoneNumberCountryCode() { + return phone_number_country_code; +} diff --git a/clients/shared/LocaleUtils.android.h b/clients/shared/LocaleUtils.android.h new file mode 100644 index 0000000..0047c32 --- /dev/null +++ b/clients/shared/LocaleUtils.android.h @@ -0,0 +1,9 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_LOCALE_UTILS_ANDROID_H +#define VIEWFINDER_LOCALE_UTILS_ANDROID_H + +extern string phone_number_country_code; + +#endif // VIEWFINDER_LOCALE_UTILS_ANDROID_H diff --git a/clients/shared/LocaleUtils.h b/clients/shared/LocaleUtils.h new file mode 100644 index 0000000..ef69b27 --- /dev/null +++ b/clients/shared/LocaleUtils.h @@ -0,0 +1,8 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#import "Utils.h" + +// Returns a country code to be used for phone number input, in the form used by the phonenumbers +// library (uppercase two-character ISO codes) +string GetPhoneNumberCountryCode(); diff --git a/clients/shared/LocaleUtils.ios.mm b/clients/shared/LocaleUtils.ios.mm new file mode 100644 index 0000000..93c727a --- /dev/null +++ b/clients/shared/LocaleUtils.ios.mm @@ -0,0 +1,38 @@ +// Copryright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#import +#import +#import +#import "LocaleUtils.h" +#import "StringUtils.h" + +namespace { + +std::once_flag once; +string phone_number_country_code; + +} // namespace + +string GetPhoneNumberCountryCode() { + std::call_once(once, [] { + // If there is a cellular provider, use its country code. + CTTelephonyNetworkInfo* info = [CTTelephonyNetworkInfo new]; + const string from_carrier = + ToUppercase(ToString(info.subscriberCellularProvider.isoCountryCode)); + if (!from_carrier.empty()) { + phone_number_country_code = from_carrier; + return; + } + + // If there is no carrier info (simulator, ipod touch, etc), fall back to the system locale. + const string from_locale = ToString([[NSLocale currentLocale] objectForKey:NSLocaleCountryCode]); + if (!from_locale.empty()) { + phone_number_country_code = from_locale; + return; + } + // As a last resort, default to US. + phone_number_country_code = "US"; + }); + return phone_number_country_code; +} diff --git a/clients/shared/Location.proto b/clients/shared/Location.proto new file mode 100644 index 0000000..2e0f894 --- /dev/null +++ b/clients/shared/Location.proto @@ -0,0 +1,14 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "LocationPB"; + +message Location { + // The latitude/longitude of the breadcrumb. + optional double latitude = 1; + optional double longitude = 2; + // The accuracy of the location measurement in meters. + optional double accuracy = 3; + optional double altitude = 4; +} diff --git a/clients/shared/LocationUtils.cc b/clients/shared/LocationUtils.cc new file mode 100644 index 0000000..8d4446e --- /dev/null +++ b/clients/shared/LocationUtils.cc @@ -0,0 +1,511 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#include +#include "LazyStaticPtr.h" +#include "Location.pb.h" +#include "LocationUtils.h" +#include "Logging.h" +#include "Placemark.pb.h" +#include "STLUtils.h" +#include "StringUtils.h" + +namespace { + +const double kMeanEarthRadius = 6371 * 1000; +const double kPi = 3.14159265358979323846; + +// Abbreviates a place name if the name is >= eight characters +// and contains more than one word by taking the first letter of each +// word. Returns true if the placename was successfully abbreviated, +// and stores the result in *abbrev. +bool MaybeAbbreviatePlacename(const string& placename, string* abbrev) { + if (placename.length() >= 8) { + vector split = Split(placename, " "); + if (split.size() > 1) { + abbrev->clear(); + for (int i = 0; i < split.size(); ++i) { + abbrev->append(split[i].substr(0, 1)); + } + return true; + } + } + return false; +} + +// TODO(spencer): look into making unordered_map work, which +// would save some memory here. +class CountryAbbrevMap : public std::unordered_map { + public: + CountryAbbrevMap() { + const struct { + const char* alpha_2; + const char* alpha_3; + } kData[] = { + {"AF", "AFG"}, + {"AX", "ALA"}, + {"AL", "ALB"}, + {"DZ", "DZA"}, + {"AS", "ASM"}, + {"AD", "AND"}, + {"AO", "AGO"}, + {"AI", "AIA"}, + {"AQ", "ATA"}, + {"AG", "ATG"}, + {"AR", "ARG"}, + {"AM", "ARM"}, + {"AW", "ABW"}, + {"AU", "AUS"}, + {"AT", "AUT"}, + {"AZ", "AZE"}, + {"BS", "BHS"}, + {"BH", "BHR"}, + {"BD", "BGD"}, + {"BB", "BRB"}, + {"BY", "BLR"}, + {"BE", "BEL"}, + {"BZ", "BLZ"}, + {"BJ", "BEN"}, + {"BM", "BMU"}, + {"BT", "BTN"}, + {"BO", "BOL"}, + {"BQ", "BES"}, + {"BA", "BIH"}, + {"BW", "BWA"}, + {"BV", "BVT"}, + {"BR", "BRA"}, + {"IO", "IOT"}, + {"BN", "BRN"}, + {"BG", "BGR"}, + {"BF", "BFA"}, + {"BI", "BDI"}, + {"KH", "KHM"}, + {"CM", "CMR"}, + {"CA", "CAN"}, + {"CV", "CPV"}, + {"KY", "CYM"}, + {"CF", "CAF"}, + {"TD", "TCD"}, + {"CL", "CHL"}, + {"CN", "CHN"}, + {"CX", "CXR"}, + {"CC", "CCK"}, + {"CO", "COL"}, + {"KM", "COM"}, + {"CG", "COG"}, + {"CD", "COD"}, + {"CK", "COK"}, + {"CR", "CRI"}, + {"CI", "CIV"}, + {"HR", "HRV"}, + {"CU", "CUB"}, + {"CW", "CUW"}, + {"CY", "CYP"}, + {"CZ", "CZE"}, + {"DK", "DNK"}, + {"DJ", "DJI"}, + {"DM", "DMA"}, + {"DO", "DOM"}, + {"EC", "ECU"}, + {"EG", "EGY"}, + {"SV", "SLV"}, + {"GQ", "GNQ"}, + {"ER", "ERI"}, + {"EE", "EST"}, + {"ET", "ETH"}, + {"FK", "FLK"}, + {"FO", "FRO"}, + {"FJ", "FJI"}, + {"FI", "FIN"}, + {"FR", "FRA"}, + {"GF", "GUF"}, + {"PF", "PYF"}, + {"TF", "ATF"}, + {"GA", "GAB"}, + {"GM", "GMB"}, + {"GE", "GEO"}, + {"DE", "DEU"}, + {"GH", "GHA"}, + {"GI", "GIB"}, + {"GR", "GRC"}, + {"GL", "GRL"}, + {"GD", "GRD"}, + {"GP", "GLP"}, + {"GU", "GUM"}, + {"GT", "GTM"}, + {"GG", "GGY"}, + {"GN", "GIN"}, + {"GW", "GNB"}, + {"GY", "GUY"}, + {"HT", "HTI"}, + {"HM", "HMD"}, + {"VA", "VAT"}, + {"HN", "HND"}, + {"HK", "HKG"}, + {"HU", "HUN"}, + {"IS", "ISL"}, + {"IN", "IND"}, + {"ID", "IDN"}, + {"IR", "IRN"}, + {"IQ", "IRQ"}, + {"IE", "IRL"}, + {"IM", "IMN"}, + {"IL", "ISR"}, + {"IT", "ITA"}, + {"JM", "JAM"}, + {"JP", "JPN"}, + {"JE", "JEY"}, + {"JO", "JOR"}, + {"KZ", "KAZ"}, + {"KE", "KEN"}, + {"KI", "KIR"}, + {"KP", "PRK"}, + {"KR", "KOR"}, + {"KW", "KWT"}, + {"KG", "KGZ"}, + {"LA", "LAO"}, + {"LV", "LVA"}, + {"LB", "LBN"}, + {"LS", "LSO"}, + {"LR", "LBR"}, + {"LY", "LBY"}, + {"LI", "LIE"}, + {"LT", "LTU"}, + {"LU", "LUX"}, + {"MO", "MAC"}, + {"MK", "MKD"}, + {"MG", "MDG"}, + {"MW", "MWI"}, + {"MY", "MYS"}, + {"MV", "MDV"}, + {"ML", "MLI"}, + {"MT", "MLT"}, + {"MH", "MHL"}, + {"MQ", "MTQ"}, + {"MR", "MRT"}, + {"MU", "MUS"}, + {"YT", "MYT"}, + {"MX", "MEX"}, + {"FM", "FSM"}, + {"MD", "MDA"}, + {"MC", "MCO"}, + {"MN", "MNG"}, + {"ME", "MNE"}, + {"MS", "MSR"}, + {"MA", "MAR"}, + {"MZ", "MOZ"}, + {"MM", "MMR"}, + {"NA", "NAM"}, + {"NR", "NRU"}, + {"NP", "NPL"}, + {"NL", "NLD"}, + {"NC", "NCL"}, + {"NZ", "NZL"}, + {"NI", "NIC"}, + {"NE", "NER"}, + {"NG", "NGA"}, + {"NU", "NIU"}, + {"NF", "NFK"}, + {"MP", "MNP"}, + {"NO", "NOR"}, + {"OM", "OMN"}, + {"PK", "PAK"}, + {"PW", "PLW"}, + {"PS", "PSE"}, + {"PA", "PAN"}, + {"PG", "PNG"}, + {"PY", "PRY"}, + {"PE", "PER"}, + {"PH", "PHL"}, + {"PN", "PCN"}, + {"PL", "POL"}, + {"PT", "PRT"}, + {"PR", "PRI"}, + {"QA", "QAT"}, + {"RE", "REU"}, + {"RO", "ROU"}, + {"RU", "RUS"}, + {"RW", "RWA"}, + {"BL", "BLM"}, + {"SH", "SHN"}, + {"KN", "KNA"}, + {"LC", "LCA"}, + {"MF", "MAF"}, + {"PM", "SPM"}, + {"VC", "VCT"}, + {"WS", "WSM"}, + {"SM", "SMR"}, + {"ST", "STP"}, + {"SA", "SAU"}, + {"SN", "SEN"}, + {"RS", "SRB"}, + {"SC", "SYC"}, + {"SL", "SLE"}, + {"SG", "SGP"}, + {"SX", "SXM"}, + {"SK", "SVK"}, + {"SI", "SVN"}, + {"SB", "SLB"}, + {"SO", "SOM"}, + {"ZA", "ZAF"}, + {"GS", "SGS"}, + {"SS", "SSD"}, + {"ES", "ESP"}, + {"LK", "LKA"}, + {"SD", "SDN"}, + {"SR", "SUR"}, + {"SJ", "SJM"}, + {"SZ", "SWZ"}, + {"SE", "SWE"}, + {"CH", "CHE"}, + {"SY", "SYR"}, + {"TW", "TWN"}, + {"TJ", "TJK"}, + {"TZ", "TZA"}, + {"TH", "THA"}, + {"TL", "TLS"}, + {"TG", "TGO"}, + {"TK", "TKL"}, + {"TO", "TON"}, + {"TT", "TTO"}, + {"TN", "TUN"}, + {"TR", "TUR"}, + {"TM", "TKM"}, + {"TC", "TCA"}, + {"TV", "TUV"}, + {"UG", "UGA"}, + {"UA", "UKR"}, + {"AE", "ARE"}, + {"GB", "GBR"}, + {"US", "USA"}, + {"UM", "UMI"}, + {"UY", "URY"}, + {"UZ", "UZB"}, + {"VU", "VUT"}, + {"VE", "VEN"}, + {"VN", "VNM"}, + {"VG", "VGB"}, + {"VI", "VIR"}, + {"WF", "WLF"}, + {"EH", "ESH"}, + {"YE", "YEM"}, + {"ZM", "ZMB"}, + {"ZW", "ZWE"}, + }; + for (int i = 0; i < ARRAYSIZE(kData); ++i) { + (*this)[kData[i].alpha_2] = kData[i].alpha_3; + } + } +}; + +LazyStaticPtr kCountryAbbrevMap; + +class StateAbbrevMap : public std::unordered_map { + public: + StateAbbrevMap() { + const struct { + const char* state; + const char* abbrev; + } kData[] = { + { "Alabama", "AL" }, + { "Alaska", "AK" }, + { "Arizona", "AZ" }, + { "Arkansas", "AR" }, + { "California", "CA" }, + { "Colorado", "CO" }, + { "Connecticut", "CT" }, + { "Delaware", "DE" }, + { "Florida", "FL" }, + { "Georgia", "GA" }, + { "Hawaii", "HI" }, + { "Idaho", "ID" }, + { "Illinois", "IL" }, + { "Indiana", "IN" }, + { "Iowa", "IA" }, + { "Kansas", "KS" }, + { "Kentucky", "KY" }, + { "Louisiana", "LA" }, + { "Maine", "ME" }, + { "Maryland", "MD" }, + { "Massachusetts", "MA" }, + { "Michigan", "MI" }, + { "Minnesota", "MN" }, + { "Mississippi", "MS" }, + { "Missouri", "MO" }, + { "Montana", "MT" }, + { "Nebraska", "NE" }, + { "Nevada", "NV" }, + { "New Hampshire", "NH" }, + { "New Jersey", "NJ" }, + { "New Mexico", "NM" }, + { "New York", "NY" }, + { "North Carolina", "NC" }, + { "North Dakota", "ND" }, + { "Ohio", "OH" }, + { "Oklahoma", "OK" }, + { "Oregon", "OR" }, + { "Pennsylvania", "PA" }, + { "Rhode Island", "RI" }, + { "South Carolina", "SC" }, + { "South Dakota", "SD" }, + { "Tennessee", "TN" }, + { "Texas", "TX" }, + { "Utah", "UT" }, + { "Vermont", "VT" }, + { "Virginia", "VA" }, + { "Washington", "WA" }, + { "West Virginia", "WV" }, + { "Wisconsin", "WI" }, + { "Wyoming", "WY" }, + }; + for (int i = 0; i < ARRAYSIZE(kData); ++i) { + (*this)[kData[i].state] = kData[i].abbrev; + } + } +}; + +LazyStaticPtr kStateAbbrevMap; + +// Check whether either locality is a substring of the other. This +// handles differences in geo database names which drop the "city" +// suffix, and other similar (e.g. 'New York City' vs. 'New York'). +bool LocalityMatch(const string& l1, const string& l2) { + if (l1.find(l2) != l1.npos || l2.find(l1) != l2.npos) { + return true; + } + return false; +} + +} // namespace + + +string FormatPlacemarkWithReferencePlacemark( + const Placemark& pm, const Placemark* ref_pm, bool short_location, + PlacemarkLevel min_level, int max_parts) { + const string* parts[4] = { + (min_level <= PM_SUBLOCALITY) ? &pm.sublocality() : NULL, + &pm.locality(), + &pm.state(), + &pm.country(), + }; + + // Treat the sublocality as the locality if the locality is not present. + if (!pm.has_locality() && pm.has_sublocality() && !pm.sublocality().empty()) { + parts[1] = &pm.sublocality(); + parts[0] = NULL; + } + + if (ref_pm) { + // If the sublocality is present, but the country, state or city + // are not the same as the reference placemark and short_location + // is true, remove the sublocality. + if (short_location && parts[0] != NULL && + (*parts[3] != ref_pm->country() || + *parts[2] != ref_pm->state() || + !LocalityMatch(*parts[1], ref_pm->locality()))) { + parts[0] = NULL; + } + + if (*parts[3] == ref_pm->country()) { + // Skip the country if it's the same as the reference placemark's. + parts[3] = NULL; + if (*parts[2] == ref_pm->state() && + LocalityMatch(*parts[1], ref_pm->locality()) && + parts[0] != NULL && !parts[0]->empty()) { + // Skip the state if it's the same as the reference placemark's + // and the locality is the same as our current location...but + // only if there's a valid sublocality. + parts[2] = NULL; + } + } else { + // Skip the state if the country is not the same as the + // reference placemark's, but not if there's no locality. For + // example, "Cabrera, Maria Trinidad Sanchez, Dominican + // Republic" would become "Cabrera, Dominican Republic" if the + // reference placemark is not in the Dominican Republic. + // However, "Andhra Pradesh, India" won't just become "India". + if (parts[1] != NULL && !parts[1]->empty()) { + parts[2] = NULL; + } + } + } + + string country_abbrev; + string state_abbrev; + string locality_abbrev; + if (short_location) { + // Try to abbreviate the city name if it's the same as the + // reference placemark's. and there's also a sublocality. For + // example, ("New York City" => "NYC"). + if (parts[0] != NULL && !parts[0]->empty() && + parts[1] != NULL && !parts[1]->empty() && + ref_pm && LocalityMatch(*parts[1], ref_pm->locality())) { + if (MaybeAbbreviatePlacename(*parts[1], &locality_abbrev)) { + parts[1] = &locality_abbrev; + } + } + if (parts[2] && !parts[2]->empty()) { + if (pm.iso_country_code() == "US") { + state_abbrev = USStateAbbrev(pm.state()); + parts[2] = &state_abbrev; + } else if ((parts[0] != NULL && !parts[0]->empty()) || + (parts[1] != NULL && !parts[1]->empty())) { + // If not a US state, then abbreviate long state names with + // multiple words if we have the locality or sublocality. + if (MaybeAbbreviatePlacename(*parts[2], &state_abbrev)) { + parts[2] = &state_abbrev; + } + } + } + if (parts[3] && !parts[3]->empty()) { + country_abbrev = CountryAbbrev(pm.iso_country_code()); + parts[3] = &country_abbrev; + } + } + + string location; + const string* prev_part = NULL; + int parts_used = 0; + for (int i = min_level; i < ARRAYSIZE(parts); ++i) { + if (!parts[i] || parts[i]->empty()) { + continue; + } + if (prev_part && *prev_part == *parts[i]) { + // Skip the part if it is equal to the previous part. + continue; + } + if (!location.empty()) { + location += ", "; + } + location += *parts[i]; + prev_part = parts[i]; + parts_used++; + if (max_parts > 0 && parts_used >= max_parts) { + break; + } + } + + return location; +} + +string CountryAbbrev(const string& alpha_2) { + return FindOrDefault(*kCountryAbbrevMap, alpha_2, alpha_2); +} + +string USStateAbbrev(const string& state) { + return FindOrDefault(*kStateAbbrevMap, state, state); +} + +// Use the Haversine formula to determine the great-circle distance between 2 +// points. +double DistanceBetweenLocations(const Location& a, const Location& b) { + const double lat1 = a.latitude() * kPi / 180; + const double lng1 = a.longitude() * kPi / 180; + const double lat2 = b.latitude() * kPi / 180; + const double lng2 = b.longitude() * kPi / 180; + const double sin_dlat_2 = sin((lat2 - lat1) / 2); + const double sin_dlng_2 = sin((lng2 - lng1) / 2); + const double t = (sin_dlat_2 * sin_dlat_2) + + (cos(lat1) * cos(lat2) * sin_dlng_2 * sin_dlng_2); + return 2 * atan2(sqrt(t), sqrt(1 - t)) * kMeanEarthRadius; +} diff --git a/clients/shared/LocationUtils.h b/clients/shared/LocationUtils.h new file mode 100644 index 0000000..1c09515 --- /dev/null +++ b/clients/shared/LocationUtils.h @@ -0,0 +1,40 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_LOCATION_UTILS_H +#define VIEWFINDER_LOCATION_UTILS_H + +#include "Utils.h" + +class Location; +class Placemark; + +enum PlacemarkLevel { + PM_SUBLOCALITY, + PM_LOCALITY, + PM_STATE, + PM_COUNTRY, +}; + +// Formats the placemark into a string within the context of the +// specified breadcrumb. If 'short_location' is specified, +// abbreviations are employed and sublocalities and states are dropped +// according to co-location within locality or country. Returns the +// formatted placemark as a string. +// 'max_parts' can be used to limit the length of the returned string +// (e.g. max_parts=2 for"SoHo, New York City" instead of "SoHo, New York +// City, NY, United States"). +string FormatPlacemarkWithReferencePlacemark( + const Placemark& pm, const Placemark* ref_pm, bool short_location, + PlacemarkLevel min_level, int max_parts=-1); + +// Returns the ISO-3166-1 3 letter Alpha-3 code from the 2 +// letter Alpha-2 code. +string CountryAbbrev(const string& country); + +// Returns the 2 letter US state abbreviation. +string USStateAbbrev(const string& state); + +double DistanceBetweenLocations(const Location& a, const Location& b); + +#endif // VIEWFINDER_LOCATION_UTILS_H diff --git a/clients/shared/Logging.cc b/clients/shared/Logging.cc new file mode 100644 index 0000000..ef547c9 --- /dev/null +++ b/clients/shared/Logging.cc @@ -0,0 +1,506 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#ifdef OS_ANDROID +#import +#else // !OS_ANDROID +#import +#endif // !OS_ANDROID +#import +#import +#import "Callback.h" +#import "Compat.android.h" +#import "FileUtils.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "PathUtils.h" +#import "StringUtils.h" +#import "WallTime.h" + +namespace { + +#if TARGET_IPHONE_SIMULATOR +const int64_t kMaxLogBytes = 1000 << 20; // 1000 MB +const int64_t kMaxLogFileBytes = 100 << 20; // 100 MB +const string kLogSuffix = ""; +#else // TARGET_IPHONE_SIMULATOR +const int64_t kMaxLogBytes = 20 << 20; // 20 MB +const int64_t kMaxLogFileBytes = 100 << 10; // 100 KB +const string kLogSuffix = ".gz"; +#endif // TARGET_IPHONE_SIMULATOR + +LazyStaticPtr kLogFilenameRE = { "([^.]+)(\\..*)" }; + +pthread_once_t logging_init = PTHREAD_ONCE_INIT; + +CallbackSet1* sinks; +CallbackSet* fatals; +CallbackSet* rotates; +std::function fatal_hook; + +class FileDescriptorStreamBuf : public std::streambuf { + public: + FileDescriptorStreamBuf(int fd) + : fd_(fd), + synced_offset_(0) { + // setp() takes a pointer to the beginning of the buffer and just past its end. + setp(&buf_[0], &buf_[sizeof(buf_)]); + } + ~FileDescriptorStreamBuf() { + sync(); + } + + protected: + int sync() { + const int num = pptr() - pbase(); + if (num > 0) { + WriteStringToFD(fd_, Slice(buf_, num), true); + pbump(-num); + synced_offset_ += num; + } + return 0; + } + + int overflow(int c) { + if (c != EOF) { + sync(); + *pptr() = c; + pbump(1); + } + return c; + } + + // seekoff is a combined seek/tell method that is used in ostream::tellp(). + // We don't support seeking, but we do need to implement this for offset 0 so we can + // tell how much has been written. + // http://www.cplusplus.com/reference/ostream/ostream/tellp/ specifies + // that tellp() always calls exactly seekoff(0, ios_base::cur, ios_base::out). + std::streampos seekoff(std::streamoff off, std::ios_base::seekdir way, std::ios_base::openmode which) { + if (off != 0 || way != std::ios_base::cur || which != std::ios_base::out) { + return -1; + } + return synced_offset_ + (pptr() - pbase()); + } + + private: + int fd_; + char buf_[4096 - sizeof(int) - sizeof(std::streambuf)]; + int64_t synced_offset_; +}; + +void StderrOutput(const LogArgs& args) { + if (args.vlog) { + // Don't output VLOGs to stderr. + return; + } + + std::cerr << WallTimeFormat("%F %T:%Q", args.timestamp) + << " [" << args.pid << ":" << args.tid << "]" + << " " << args.file_line << " "; + + const char* ptr = &args.message[0]; + const char* end = ptr + args.message.size(); + bool prefix = false; + + while (ptr < end) { + if (prefix) { + prefix = false; + std::cerr.write(" ", 4); + } + + const char* lf = std::find(ptr, end, '\n'); + if (lf != end) { + prefix = true; + lf += 1; + } + + std::cerr.write(ptr, lf - ptr); + ptr = lf; + } +} + +void LoggingInit() { + sinks = new CallbackSet1; + fatals = new CallbackSet; + rotates = new CallbackSet; + Logging::AddLogSink([](const LogArgs& args) { + StderrOutput(args); + }); +} + +#ifdef DEVELOPMENT + +// Returns true if the current process is being debugged (either running under +// the debugger or has a debugger attached post facto). Apparently Apple frowns +// on the use of sysctl() in AppStore binaries, so only compile this code for +// debug builds. +bool AmIBeingDebugged() { + // Initialize mib, which tells sysctl the info we want, in this case we're + // looking for information about a specific process ID. + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = getpid(); + + kinfo_proc info; + // Initialize the flags so that, if sysctl fails for some bizarre reason, we + // get a predictable result. + info.kp_proc.p_flag = 0; + + // Call sysctl. + size_t size = sizeof(info); + CHECK_EQ(0, sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0)); + + // We're being debugged if the P_TRACED flag is set. + return (info.kp_proc.p_flag & P_TRACED) != 0; +} + +#else // !DEVELOPMENT + +bool AmIBeingDebugged() { + return false; +} + +#endif // !DEVELOPMENT + +// The file descriptor we're logging to. When we're not running under the +// debugger, this is initialized to STDERR_FILENO. When running under the +// debugger this variable is initialized to the first open log file +// descriptor. After the initialization (which takes place on the main thread +// before locking is an issue), we use dup2() to clone the file descriptor for +// new log files on top of the the existing log_fd. +int log_fd = -1; +ostream* log_stream = NULL; +int64_t log_file_size; +string log_file_base; +string log_file_name; +int log_file_id; + +int NewLogFile() { + const string base = NewLogFilename(""); + if (base != log_file_base) { + log_file_id = 0; + } else { + ++log_file_id; + } + for (; ; ++log_file_id) { + const string name = (log_file_id == 0) ? + Format("%s.log", base) : + Format("%s.%d.log", base, log_file_id); + const string path = JoinPath(LoggingDir(), name); + const int new_fd = FileCreate(path); + if (new_fd < 0) { + continue; + } + log_file_base = base; + log_file_name = name; + const string s = Format("init: log file: %s\n", name); + WriteStringToFD(new_fd, s, true); + return new_fd; + } + return -1; +} + +Mutex garbage_collect_mu; + +// Iterate over the log files from newest to oldest, deleting files when their +// cumulative size exceeds kMaxLogBytes. +void GarbageCollectLogs() { + MutexLock l(&garbage_collect_mu); + + const string queue_dir = LoggingQueueDir(); + vector old_logs; + DirList(queue_dir, &old_logs); + + // TODO(pmattis): Is the sort necessary? + std::sort(old_logs.begin(), old_logs.end()); + std::reverse(old_logs.begin(), old_logs.end()); + int64_t cumulative_size = 0; + + for (int i = 0; i < old_logs.size(); ++i) { + const int file_size = FileSize(JoinPath(queue_dir, old_logs[i])); + if (file_size < 0) { + continue; + } + cumulative_size += file_size; + if (i >= 1 && cumulative_size >= kMaxLogBytes) { + LOG("init: deleting: %s", old_logs[i]); + FileRemove(JoinPath(queue_dir, old_logs[i])); + } + } +} + +void CompressAndRename( + const string& src, const string& dest_dir, const string& dest_name) { +#if TARGET_IPHONE_SIMULATOR + FileRename(src, JoinPath(dest_dir, dest_name)); +#else // TARGET_IPHONE_SIMULATOR + if (FileSize(src) > kMaxLogFileBytes * 2) { + // Something's gone wrong with log rotation; just delete the log + // instead of trying to read it into memory. + FileRemove(src); + return; + } + string raw = ReadFileToString(src); + if (raw.empty()) { + return; + } + // Gzip the data. + string gzip = GzipEncode(raw); + raw.clear(); + // Write to tmp/. + const string tmp_path = JoinPath(TmpDir(), dest_name); + WriteStringToFile(tmp_path, gzip); + gzip.clear(); + // Move from tmp/ to /. + FileRename(tmp_path, JoinPath(dest_dir, dest_name)); + // Remove file. + FileRemove(src); +#endif // TARGET_IPHONE_SIMULATOR +} + +void MaybeRotateLog() { + if (log_file_size < kMaxLogFileBytes) { + return; + } + const string old_file_name = log_file_name; + const int new_fd = NewLogFile(); + log_stream->flush(); + const int err = dup2(new_fd, log_fd); + close(new_fd); + if (err < 0) { + return; + } + if (!old_file_name.empty()) { + dispatch_background([old_file_name] { + CompressAndRename(JoinPath(LoggingDir(), old_file_name), + LoggingQueueDir(), old_file_name + kLogSuffix); + GarbageCollectLogs(); + rotates->Run(); + }); + } + log_file_size = FileSize(log_fd); +} + +} // namespace + +const char* LogFormatFileLine(const char* file) { + if (!file) { + return NULL; + } + const char* p = strrchr(file, '/'); + if (p) file = p + 1; + return file; +} + +LogStream::LogStream(string* output) + : strm_(this), + output_(output) { + setp(&buf_[0], &buf_[sizeof(buf_) - 1]); +} + +LogStream::~LogStream() { + sync(); + if (*output_->rbegin() != '\n') { + output_->append("\n", 1); + } +} + +int LogStream::sync() { + const int num = pptr() - pbase(); + if (num > 0) { + output_->append(buf_, num); + pbump(-num); + } + return 0; +} + +int LogStream::overflow(int c) { + if (c != EOF) { + *pptr() = c; + pbump(1); + sync(); + } + return c; +} + +LogArgs::LogArgs(const char* fl, bool v) + : file_line(LogFormatFileLine(fl)), + timestamp(WallTime_Now()), + pid(getpid()), +#ifdef OS_ANDROID + tid(gettid()), +#else + tid(pthread_mach_thread_np(pthread_self())), +#endif + vlog(v) { +} + +LogMessage::Helper::~Helper() { + sinks->Run(args); + if (die) { + fatals->Run(); + abort(); + } +} + +LogMessage::LogMessage( + const char* file_line, bool die, bool vlog) + : helper_(die, vlog, file_line), + stream_(&helper_.args.message) { +} + +LogMessage::~LogMessage() { + if (helper_.die) { + if (fatal_hook) { + fatal_hook(stream_); + } + stream_ << "\n"; + } +} + +void Logging::AddLogSink(const LogSink& sink) { + sinks->Add(sink); +} + +int Logging::AddLogFatal(const LogCallback& callback) { + return fatals->Add(callback); +} + +int Logging::AddLogRotate(const LogCallback& callback) { + return rotates->Add(callback); +} + +void Logging::RemoveLogFatal(int id) { + fatals->Remove(id); +} + +void Logging::RemoveLogRotate(int id) { + rotates->Remove(id); +} + +void Logging::SetFatalHook(const LogFatalCallback& callback) { + fatal_hook = callback; +} + +void Logging::InitFileLogging() { + // TODO(ben): stderr on a device that is not being debugged is weird; writing to it with write() works + // but through a c++ stream doesn't (manual flushing doesn't seem to work). Now that we're writing through + // a stream, we need to turn off our stderr integration until we figure out what's going on. + /*if (!AmIBeingDebugged()) { + // We're not being run under a debugger. Redirect stderr to our log + // file. When we're running under a debugger we want to leave stderr alone + // because it outputs to the debugger console. + log_fd = STDERR_FILENO; + log_stream = &std::cerr; + log_file_size = kMaxLogFileBytes; + }*/ + + DirCreate(LibraryDir()); + const string logging_dir = LoggingDir(); + const string queue_dir = LoggingQueueDir(); + + DirCreate(logging_dir); + DirCreate(queue_dir); + + { + // Move any existing log files into the queue directory. + vector old_logs; + DirList(logging_dir, &old_logs); + dispatch_background([old_logs, logging_dir, queue_dir] { + for (int i = 0; i < old_logs.size(); ++i) { + const string src = JoinPath(logging_dir, old_logs[i]); + if (src == queue_dir) { + // Skip moving the queue dir into itself (which would fail anyways). + continue; + } + CompressAndRename(src, queue_dir, old_logs[i] + kLogSuffix); + } + GarbageCollectLogs(); + }); + } + + if (!AmIBeingDebugged()) { + // We're not being run under a debugger. Clear the stderr log sink to + // prevent double-writing of log messages to the same file. + sinks->Clear(); + } + + if (log_fd < 0) { + log_fd = NewLogFile(); + log_file_size = FileSize(log_fd); + log_stream = new ostream(new FileDescriptorStreamBuf(log_fd)); + } + + Logging::AddLogSink([](const LogArgs& args) { + // Note, we're protected by sinks->mu_ here. The 3rd parameter to + // WallTimeFormat specifies UTC time. + const std::ostream::pos_type old_size = log_stream->tellp(); + *log_stream << WallTimeFormat("%F %T:%Q", args.timestamp, false) + << " [" << args.pid << ":" << args.tid << "]" + << (args.file_line ? " " : "") + << (args.file_line ? args.file_line : "") + << " " << args.message; + log_file_size += log_stream->tellp() - old_size; + + MaybeRotateLog(); + }); + + // Perform an initial log rotation. This will create the initial log file if + // !AmIBeingDebugged(). + MaybeRotateLog(); +} + +void Logging::Init() { + pthread_once(&logging_init, &LoggingInit); +} + +struct ScopedLogSink::Impl { + CallbackSet1 old_sinks; +}; + +ScopedLogSink::ScopedLogSink() + : impl_(new Impl) { + sinks->Swap(&impl_->old_sinks); + sinks->Add([this](const LogArgs& args) { + output_ += Format("%s [%d:%d] %s %s", + WallTimeFormat("%F %T:%Q", args.timestamp), + args.pid, args.tid, args.file_line ? args.file_line : "", + args.message); + }); +} + +ScopedLogSink::~ScopedLogSink() { + sinks->Swap(&impl_->old_sinks); + delete impl_; +} + +string NewLogFilename(const string& suffix) { + // The 3rd parameter to WallTimeFormat specifies UTC time. + return Format("%s-%s%s", + WallTimeFormat("%F-%H-%M-%S.%Q", WallTime_Now(), false), + AppVersion(), suffix); +} + +bool ParseLogFilename( + const string& filename, WallTime* timestamp, string* suffix) { + string datetime; + if (!RE2::FullMatch(filename, *kLogFilenameRE, &datetime, suffix)) { + return false; + } + struct tm t; + memset(&t, 0, sizeof(t)); + if (!strptime(datetime.c_str(), "%F-%H-%M-%S", &t)) { + return false; + } + t.tm_isdst = -1; + *timestamp = timegm(&t); + return true; +} diff --git a/clients/shared/Logging.h b/clients/shared/Logging.h new file mode 100644 index 0000000..bea1cc0 --- /dev/null +++ b/clients/shared/Logging.h @@ -0,0 +1,354 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_LOGGING_H +#define VIEWFINDER_LOGGING_H + +#import "Format.h" +#import "Utils.h" +#import "WallTime.h" + +const char* LogFormatFileLine(const char* file_line); + +class LogStream : private std::streambuf { + typedef Formatter::Arg Arg; + + public: + explicit LogStream(string* output); + ~LogStream(); + + LogStream& operator<<(ostream& (*val)(ostream&)) { + strm_ << val; + return *this; + } + + template + LogStream& operator<<(const T &val) { + strm_ << val; + return *this; + } + + LogStream& operator()(const char* fmt) { + Format(fmt).Apply(strm_); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0) { + const Arg* const args[] = { &a0 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1) { + const Arg* const args[] = { &a0, &a1 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2) { + const Arg* const args[] = { &a0, &a1, &a2 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3) { + const Arg* const args[] = { &a0, &a1, &a2, &a3 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8, + &a9 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9, + const Arg& a10) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8, + &a9, &a10 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + LogStream& operator()(const char* fmt, const Arg& a0, const Arg& a1, + const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5, + const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9, + const Arg& a10, const Arg& a11) { + const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8, + &a9, &a10, &a11 }; + Format(fmt).Apply(strm_, args, ARRAYSIZE(args)); + return *this; + } + + private: + int sync(); + int overflow(int c); + + private: + ostream strm_; + string* const output_; + char buf_[256]; +}; + +class LogStreamVoidify { + public: + LogStreamVoidify() { } + void operator&(LogStream&) { } +}; + +struct LogArgs { + LogArgs(const char* file_line, bool v); + + string message; + const char* const file_line; + const double timestamp; + const int pid; + const int tid; + const bool vlog; +}; + +typedef std::function LogSink; + +class LogMessage { + struct Helper { + Helper(bool d, bool r, const char* file_line) + : die(d), + args(file_line, r) { + } + ~Helper(); + + const bool die; + LogArgs args; + }; + + public: + LogMessage(const char* file_line, bool die, bool vlog); + ~LogMessage(); + + LogStream& stream() { return stream_; } + + private: + Helper helper_; + LogStream stream_; +}; + +class Logging { + typedef std::function LogCallback; + typedef std::function LogFatalCallback; + + public: + class Initializer { + public: + Initializer() { + Logging::Init(); + } + }; + + public: + static void AddLogSink(const LogSink& sink); + static int AddLogFatal(const LogCallback& callback); + static int AddLogRotate(const LogCallback& callback); + static void RemoveLogFatal(int id); + static void RemoveLogRotate(int id); + static void SetFatalHook(const LogFatalCallback& callback); + static void InitFileLogging(); + + private: + static void Init(); +}; + +static Logging::Initializer kLoggingInitializer; + +class ScopedLogSink { + struct Impl; + + public: + ScopedLogSink(); + ~ScopedLogSink(); + + string output() const { return output_; } + + private: + Impl* impl_; + string output_; +}; + +#define LOG_FILE_LINE3(x) #x +#define LOG_FILE_LINE2(x) LOG_FILE_LINE3(x) +#define LOG_FILE_LINE __FILE__ ":" LOG_FILE_LINE2(__LINE__) ":" + +#define LOG \ + LogMessage(LOG_FILE_LINE, false, false).stream() +// VLOG is like LOG, except it is only output to a file, not to stderr. +#define VLOG \ + LogMessage(LOG_FILE_LINE, false, true).stream() +#define DIE \ + LogMessage(LOG_FILE_LINE, true, false).stream() +#define CHECK(cond) \ + (cond) ? (void) 0 : \ + LogStreamVoidify() & \ + LogMessage(LOG_FILE_LINE, true, false).stream() \ + << "check failed: " << #cond + +#ifdef DEBUG +#define DCHECK(cond) CHECK(cond) +#else +#define DCHECK(cond) \ + (cond) ? (void) 0 : \ + LogStreamVoidify() & \ + LogMessage(LOG_FILE_LINE, false, false).stream() \ + << "dcheck failed: " << #cond +#endif + +// Function is overloaded for integral types to allow static const +// integrals declared in classes and not defined to be used as arguments to +// CHECK* macros. It's not encouraged though. +template +inline const T& GetReferenceableValue(const T& t) { return t; } +inline char GetReferenceableValue(char t) { return t; } +inline unsigned char GetReferenceableValue(unsigned char t) { return t; } +inline signed char GetReferenceableValue(signed char t) { return t; } +inline short GetReferenceableValue(short t) { return t; } +inline unsigned short GetReferenceableValue(unsigned short t) { return t; } +inline int GetReferenceableValue(int t) { return t; } +inline unsigned int GetReferenceableValue(unsigned int t) { return t; } +inline long GetReferenceableValue(long t) { return t; } +inline unsigned long GetReferenceableValue(unsigned long t) { return t; } +inline long long GetReferenceableValue(long long t) { return t; } +inline unsigned long long GetReferenceableValue(unsigned long long t) { + return t; +} + +// Helper functions for CHECK_OP macro. +// The (int, int) specialization works around the issue that the compiler +// will not instantiate the template version of the function on values of +// unnamed enum type - see comment below. +#define DEFINE_CHECK_OP_IMPL(name, op) \ + template \ + inline string* Check##name##Impl(const T1& v1, const T2& v2, \ + const char* names) { \ + if (v1 op v2) return NULL; \ + else return new string(Format("%s (%s vs %s)", names, v1, v2)); \ + } \ + inline string* Check##name##Impl(int v1, int v2, const char* names) { \ + return Check##name##Impl(v1, v2, names); \ + } + +// Use _EQ, _NE, _LE, etc. in case the simpler names EQ, NE, LE, etc are +// already defined. This happens if, for example, those are used as token names +// in a yacc grammar. +DEFINE_CHECK_OP_IMPL(_EQ, ==) +DEFINE_CHECK_OP_IMPL(_NE, !=) +DEFINE_CHECK_OP_IMPL(_LE, <=) +DEFINE_CHECK_OP_IMPL(_LT, < ) +DEFINE_CHECK_OP_IMPL(_GE, >=) +DEFINE_CHECK_OP_IMPL(_GT, > ) +#undef DEFINE_CHECK_OP_IMPL + +// In debug mode, avoid constructing CheckOpStrings if possible, +// to reduce the overhead of CHECK statments by 2x. +// Real DCHECK-heavy tests have seen 1.5x speedups. +#define CHECK_OP(name, op, val1, val2) \ + while (string* _result = \ + Check##name##Impl( \ + GetReferenceableValue(val1), \ + GetReferenceableValue(val2), \ + #val1 " " #op " " #val2)) \ + LogMessage(LOG_FILE_LINE, true, false).stream() \ + << "check failed: " << *_result + +// Equality/Inequality checks - compare two values, and log a FATAL message +// including the two values when the result is not as expected. The values +// must have operator<<(ostream, ...) defined. +// +// You may append to the error message like so: +// CHECK_NE(1, 2) << ": The world must be ending!"; +// +// We are very careful to ensure that each argument is evaluated exactly +// once, and that anything which is legal to pass as a function argument is +// legal here. In particular, the arguments may be temporary expressions +// which will end up being destroyed at the end of the apparent statement, +// for example: +// CHECK_EQ(string("abc")[1], 'b'); +// +// WARNING: These don't compile correctly if one of the arguments is a pointer +// and the other is NULL. To work around this, simply static_cast NULL to the +// type of the desired pointer. + +#define CHECK_EQ(val1, val2) CHECK_OP(_EQ, ==, val1, val2) +#define CHECK_NE(val1, val2) CHECK_OP(_NE, !=, val1, val2) +#define CHECK_LE(val1, val2) CHECK_OP(_LE, <=, val1, val2) +#define CHECK_LT(val1, val2) CHECK_OP(_LT, < , val1, val2) +#define CHECK_GE(val1, val2) CHECK_OP(_GE, >=, val1, val2) +#define CHECK_GT(val1, val2) CHECK_OP(_GT, > , val1, val2) +#define CHECK_NEAR(val1, val2) CHECK_OP(_LT, < , fabs((val1) - (val2)), \ + std::numeric_limits::epsilon()) + +#ifdef DEBUG +#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) +#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) +#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) +#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) +#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) +#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) +#define DCHECK_NEAR(val1, val2) CHECK_NEAR(val1, val2) +#else // DEBUG +#define DCHECK_EQ(val1, val2) while(false) CHECK_EQ(val1, val2) +#define DCHECK_NE(val1, val2) while(false) CHECK_NE(val1, val2) +#define DCHECK_LE(val1, val2) while(false) CHECK_LE(val1, val2) +#define DCHECK_LT(val1, val2) while(false) CHECK_LT(val1, val2) +#define DCHECK_GE(val1, val2) while(false) CHECK_GE(val1, val2) +#define DCHECK_GT(val1, val2) while(false) CHECK_GT(val1, val2) +#define DCHECK_NEAR(val1, val2) while(false) CHECK_NEAR(val1, val2) +#endif // DEBUG + +string NewLogFilename(const string& suffix); +bool ParseLogFilename(const string& filename, WallTime* timestamp, string* suffix); + +#endif // VIEWFINDER_LOGGING_H diff --git a/clients/shared/Mutex.cc b/clients/shared/Mutex.cc new file mode 100644 index 0000000..6fdb3e5 --- /dev/null +++ b/clients/shared/Mutex.cc @@ -0,0 +1,58 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "Mutex.h" + +Mutex::~Mutex() { + pthread_mutex_destroy(&mu_); + if (cond_) { + pthread_cond_destroy(cond_); + delete cond_; + } +} + +void Mutex::Wait(const Predicate& pred) { + if (pred()) { + return; + } + if (!cond_) { + cond_ = new pthread_cond_t; + pthread_cond_init(cond_, NULL); + } + do { + pthread_cond_wait(cond_, &mu_); + } while (!pred()); +} + +bool Mutex::TimedWait(WallTime max_wait, const Predicate& pred) { + if (pred()) { + return true; + } + const WallTime end_time = WallTime_Now() + max_wait; + if (!cond_) { + cond_ = new pthread_cond_t; + pthread_cond_init(cond_, NULL); + } + timespec ts; + ts.tv_sec = static_cast(end_time); + ts.tv_nsec = static_cast((end_time - ts.tv_sec) * 1e9); + do { + if (pthread_cond_timedwait(cond_, &mu_, &ts) != 0) { + return false; + } + } while (!pred()); + return true; +} + + +void Barrier::Signal() { + MutexLock l(&mu_); + CHECK_GT(count_, 0); + --count_; +} + +void Barrier::Wait() { + MutexLock l(&mu_); + CHECK_GE(count_, 0); + mu_.Wait([this] { return count_ == 0; }); +} diff --git a/clients/shared/Mutex.h b/clients/shared/Mutex.h new file mode 100644 index 0000000..fd7aac5 --- /dev/null +++ b/clients/shared/Mutex.h @@ -0,0 +1,100 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_MUTEX_H +#define VIEWFINDER_MUTEX_H + +#import +#import +#import "Logging.h" +#import "Utils.h" +#import "WallTime.h" + +class Mutex { + typedef std::function Predicate; + + public: + Mutex() + : cond_(NULL) { + pthread_mutex_init(&mu_, NULL); + } + ~Mutex(); + + void Lock() { + pthread_mutex_lock(&mu_); + } + bool TryLock() { + return pthread_mutex_trylock(&mu_) == 0; + } + void Unlock() { + if (cond_) { + pthread_cond_broadcast(cond_); + } + pthread_mutex_unlock(&mu_); + } + + void Wait(const Predicate& pred); + bool TimedWait(WallTime max_wait, const Predicate& pred); + + // In debug builds, fail a CHECK if the lock is not held. This does not + // guarantee that the lock is held by the current thread, but it allows + // unittests to catch cases where the lock is not held at all. + void AssertHeld() { +#ifdef DEBUG + const int result = pthread_mutex_trylock(&mu_); + if (result == 0) { + pthread_mutex_unlock(&mu_); + DIE("mutex was not held"); + } +#endif // DEBUG + } + + private: + pthread_mutex_t mu_; + pthread_cond_t* cond_; + + private: + // Catch the error of writing Mutex when intending MutexLock. + Mutex(Mutex*) = delete; + // Disallow "evil" constructors + Mutex(const Mutex&) = delete; + void operator=(const Mutex&) = delete; +}; + +class MutexLock { + public: + explicit MutexLock(Mutex *mu) + : mu_(mu) { + mu_->Lock(); + } + ~MutexLock() { + mu_->Unlock(); + } + + private: + Mutex * const mu_; + + private: + // Disallow "evil" constructors + MutexLock(const MutexLock&); + void operator=(const MutexLock&); +}; + +// Catch bug where variable name is omitted, e.g. MutexLock (&mu); +// #define MutexLock(x) COMPILE_ASSERT(0, mutex_lock_decl_missing_var_name) + +class Barrier { + public: + Barrier(int n) + : count_(n) { + } + + void Signal(); + void Wait(); + + private: + Mutex mu_; + int count_; +}; + +#endif // VIEWFINDER_MUTEX_H diff --git a/clients/shared/NetworkManager.cc b/clients/shared/NetworkManager.cc new file mode 100644 index 0000000..5e02057 --- /dev/null +++ b/clients/shared/NetworkManager.cc @@ -0,0 +1,3932 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#import "ActivityTable.h" +#import "Analytics.h" +#import "AppState.h" +#import "AsyncState.h" +#import "ContactManager.h" +#import "DB.h" +#import "Defines.h" +#import "DigestUtils.h" +#import "FileUtils.h" +#import "IdentityManager.h" +#import "Logging.h" +#import "NetworkManager.h" +#import "NetworkQueue.h" +#import "NotificationManager.h" +#import "PathUtils.h" +#import "Server.pb.h" +#import "ServerId.h" +#import "ServerUtils.h" +#import "StringUtils.h" +#import "SubscriptionManager.h" +#import "Timer.h" + +const WallTime NetworkManager::kMinBackoffDelay = 1; + +namespace { + +// Disable NETLOG statements in APPSTORE builds as they contain Personally +// Identifiable Information. +#ifdef APPSTORE +#define NETLOG if (0) VLOG +#else +#define NETLOG VLOG +#endif + +const string kAccessCodeKey = DBFormat::metadata_key("access_code/"); +const string kPushDeviceTokenKey = DBFormat::metadata_key("apn_device_token"); +const string kQueryFollowedDoneKey = + DBFormat::metadata_key("query_followed_done_key"); +const string kQueryFollowedLastKey = + DBFormat::metadata_key("query_followed_last_key"); + +const WallTime kMaxBackoffDelay = 60 * 10; +const WallTime kLogUploadInterval = 10; +const int kQueryContactsLimit = 1000; +const int kQueryUsersLimit = 500; +const int kQueryObjectsLimit = 200; +const int kQueryEpisodesLimit = 100; +const int kQueryFollowedLimit = 100; +const int kQueryNotificationsLimit = 100; +const int kQueryViewpointsLimit = 100; +const WallTime kPingPeriodDefault = 12 * 60 * 60; +const WallTime kPingPeriodFast = 10 * 60; +const WallTime kProspectiveUserCreationDelay = 2; +// For maximum compatibility with uncooperative proxies (e.g. hotel wifi), this should be under a minute. +const WallTime kQueryNotificationsMaxLongPoll = 58; +const WallTime kQueryNotificationsMaxRetryAfter = 3600; + +#if defined(APPSTORE) && !defined(ENTERPRISE) +const int kUploadLogOptOutGracePeriod = 600; +#else +const int kUploadLogOptOutGracePeriod = 0; +#endif + +const string kJsonContentType = "application/json"; +const string kJpegContentType = "image/jpeg"; +const string kOctetStreamContentType = "application/octet-stream"; + +const string kDefaultNetworkErrorMessage = + "The network is unavailable. Please try again later."; + +const string kDefaultLoginErrorMessage = + "Your Viewfinder login failed. Please try again later."; + +const string kDefaultVerifyErrorMessage = + "Couldn't verify your identity…"; + +const string kDefaultChangePasswordErrorMessage = + "Couldn't change your password…"; + +const string kDownloadBenchmarkURLPrefix = "https://public-ro-viewfinder-co.s3.amazonaws.com/"; +const string kDownloadBenchmarkFiles[] = { + "10KB.test", + "50KB.test", + "100KB.test", + "200KB.test", + "500KB.test", + "1MB.test", + "2MB.test", +}; + +string FormatUrl(AppState* state, const string& path) { + return Format("%s://%s:%s%s", + state->server_protocol(), + state->server_host(), + state->server_port(), + path); +} + +string FormatRequest(const JsonDict& dict, int min_required_version = 0, + bool synchronous = false) { + JsonDict headers_dict = JsonDict("version", AppState::protocol_version()); + if (min_required_version > 0) { + headers_dict.insert("min_required_version", min_required_version); + } + if (synchronous) { + headers_dict.insert("synchronous", synchronous); + } + + JsonDict req_dict = dict; + req_dict.insert("headers", headers_dict); + return req_dict.Format(); +} + +string FormatRequest(const JsonDict& dict, const OpHeaders& op_headers, + AppState* state, int min_required_version = 0) { + JsonDict headers_dict = JsonDict("version", AppState::protocol_version()); + if (min_required_version > 0) { + headers_dict.insert("min_required_version", min_required_version); + } + if (op_headers.has_op_id()) { + headers_dict.insert( + "op_id", EncodeOperationId(state->device_id(), op_headers.op_id())); + } + if (op_headers.has_op_timestamp()) { + headers_dict.insert("op_timestamp", op_headers.op_timestamp()); + } + JsonDict req_dict = dict; + req_dict.insert("headers", headers_dict); + return req_dict.Format(); +} + +JsonDict FormatDeviceDict(AppState* state) { + JsonDict device({ + { "os", state->device_os() }, + { "platform", state->device_model() }, + { "version", AppVersion() } + }); + if (!state->device_name().empty()) { + device.insert("name", state->device_name()); + } + if (state->device_id() != 0) { + device.insert("device_id", state->device_id()); + } + + string push_token; + if (state->db()->Get(kPushDeviceTokenKey, &push_token)) { + device.insert("push_token", push_token); + } + + device.insert("device_uuid", state->device_uuid()); + device.insert("language", state->locale_language()); + device.insert("country", state->locale_country()); + if (!state->test_udid().empty()) { + device.insert("test_udid", state->test_udid()); + } + + return device; +} + +// Create an activity dictionary suitable for passing to the server with ops +// that must create an activity. +const JsonDict FormatActivityDict(const ActivityHandle& ah) { + return JsonDict({ + { "activity_id", ah->activity_id().server_id() }, + { "timestamp", ah->timestamp() } + }); +} + +// Creates a new activity server id using the specified local id and timestamp. +const JsonDict FormatActivityDict( + AppState* state, int64_t local_id, WallTime timestamp) { + const string activity_id = EncodeActivityId( + state->device_id(), local_id, timestamp); + return JsonDict({ + { "activity_id", activity_id }, + { "timestamp", timestamp } + }); +} + +JsonDict FormatAccountSettingsDict(AppState* state) { + // TODO: add 'email_alerts' field when settable on the client. + JsonDict account_settings; + + vector storage_options; + // We use the raw "cloud storage" toggle since we care about user-specified + // settings, not additional logic. + if (state->cloud_storage()) { + storage_options.push_back("use_cloud"); + } + if (state->store_originals()) { + storage_options.push_back("store_originals"); + } + // We need to specify 'storage_options' even if all are off, + // otherwise the backend would never know. + account_settings.insert("storage_options", + JsonArray(storage_options.size(), [&](int i) { + return storage_options[i]; + })); + + return account_settings; +} + +const char* PhotoURLSuffix(NetworkQueue::PhotoType type) { + switch (type) { + case NetworkQueue::THUMBNAIL: + return ".t"; + case NetworkQueue::MEDIUM: + return ".m"; + case NetworkQueue::FULL: + return ".f"; + case NetworkQueue::ORIGINAL: + return ".o"; + } + return ""; +} + +const char* PhotoTypeName(NetworkQueue::PhotoType type) { + switch (type) { + case NetworkQueue::THUMBNAIL: + return "thumbnail"; + case NetworkQueue::MEDIUM: + return "medium"; + case NetworkQueue::FULL: + return "full"; + case NetworkQueue::ORIGINAL: + return "original"; + } + return ""; +} + +} // namespace + +class AddFollowersRequest : public NetworkRequest { + public: + AddFollowersRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u), + needs_invalidate_(false) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d({ + { "viewpoint_id", u->viewpoint->id().server_id() }, + { "activity", FormatActivityDict(u->activity) }, + { "contacts", JsonArray(u->contacts.size(), [&](int i) { + JsonDict d; + const ContactMetadata& c = u->contacts[i]; + if (c.has_primary_identity()) { + d.insert("identity", c.primary_identity()); + } + if (c.has_user_id()) { + d.insert("user_id", c.user_id()); + } else { + // If we upload followers without user ids (prospective users), they will have user ids + // assigned by this operation. The DayTable does not display followers that do not yet + // have user ids, so we need to fetch notifications once this is done. + needs_invalidate_ = true; + } + if (c.has_name()) { + d.insert("name", c.name()); + } + return d; + }) } + }); + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: add followers: %s", json); + SendPost(FormatUrl(state(), "/service/add_followers"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: add_followers error: %s", e); + state()->analytics()->NetworkAddFollowers(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkAddFollowers(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: add_followers error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + const NetworkQueue::UploadActivity* u = upload_; + LOG("network: added %d contact%s: %.03f", + u->contacts.size(), Pluralize(u->contacts.size()), + timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + + if (needs_invalidate_) { + AppState* const s = state(); + dispatch_after_main(kProspectiveUserCreationDelay, [s] { + DBHandle updates = s->NewDBTransaction(); + s->notification_manager()->Invalidate(updates); + updates->Commit(); + }); + } + } + return true; + } + + private: + const NetworkQueue::UploadActivity* const upload_; + bool needs_invalidate_; +}; + +class AuthRequest : public NetworkRequest { + public: + AuthRequest(NetworkManager* net) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH) { + } + + protected: + void SendAuth(const string& url, const JsonDict& auth, + bool include_device_info = true) { + JsonDict augmented_auth = auth; + if (include_device_info) { + augmented_auth.insert("device", FormatDeviceDict(state())); + } + + const string json = FormatRequest(augmented_auth); + NETLOG("network: auth: %s\n%s", url, json); + SendPost(url, json, kJsonContentType); + } + + void SendAuth(const string& url) { + SendAuth(url, JsonDict()); + } + + void HandleError(const string& e) { + LOG("network: auth error: %s", e); + } + + bool HandleDone(int status_code) { + AuthResponse a; + if (!ParseAuthResponse(&a, data_)) { + LOG("network: unable to parse auth response: %s", data_); + return false; + } + return HandleDone(a); + } + + bool HandleDone(const AuthResponse& a) { + // LOG("network: auth: %s", a); + const int64_t user_id = a.has_user_id() ? a.user_id() : state()->user_id(); + const int64_t device_id = a.has_device_id() ? a.device_id() : state()->device_id(); + state()->SetUserAndDeviceId(user_id, device_id); + if (state()->is_registered()) { + net_->AuthDone(); + } + return true; + } +}; + +class AuthViewfinderRequest : public AuthRequest { + public: + AuthViewfinderRequest( + NetworkManager* net, const string& endpoint, const string& identity, + const string& password, const string& first, const string& last, + const string& name, bool error_if_linked, + const NetworkManager::AuthCallback& done) + : AuthRequest(net), + endpoint_(endpoint), + identity_(identity), + password_(password), + first_(first), + last_(last), + name_(name), + error_if_linked_(error_if_linked), + done_(done) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict auth; + bool include_device_info = true; + if (endpoint_ == AppState::kMergeTokenEndpoint) { + LOG("network: %s viewfinder, identity=\"%s\"", endpoint_, identity_); + auth.insert("identity", identity_); + auth.insert("error_if_linked", error_if_linked_); + include_device_info = false; + } else { + JsonDict auth_info("identity", identity_); + if (endpoint_ == AppState::kRegisterEndpoint) { + LOG("network: %s viewfinder, identity=\"%s\", first=\"%s\", " + "last=\"%s\", name=\"%s\"", + endpoint_, identity_, first_, last_, name_); + if (!name_.empty()) { + auth_info.insert("name", name_); + } + if (!first_.empty()) { + auth_info.insert("given_name", first_); + } + if (!last_.empty()) { + auth_info.insert("family_name", last_); + } + } else { + LOG("network: %s viewfinder, identity=\"%s\"", endpoint_, identity_); + } + if (endpoint_ == AppState::kRegisterEndpoint || + endpoint_ == AppState::kLoginEndpoint) { + if (!password_.empty()) { + auth_info.insert("password", password_); + } + } + auth.insert("auth_info", auth_info); + } + SendAuth(FormatUrl(state(), Format("/%s/viewfinder", endpoint_)), + auth, include_device_info); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + state()->analytics()->NetworkAuthViewfinder(0, timer_.Get()); + AuthRequest::HandleError(e); + done_(-1, ErrorResponse::UNKNOWN, e); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkAuthViewfinder(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: auth viewfinder error: %d status: %s\n%s", + status_code, url(), data_); + // Note, unlike most other network requests, we do not retry on 5xx + // errors and instead pass back the error to the caller so an error can + // be displayed to the user. + ErrorResponse err; + if (!ParseErrorResponse(&err, data_)) { + done_(status_code, ErrorResponse::UNKNOWN, kDefaultLoginErrorMessage); + } else { + done_(status_code, err.error().error_id(), err.error().text()); + } + return true; + } + + AuthResponse a; + if (!ParseAuthResponse(&a, data_)) { + LOG("network: unable to parse auth response: %s", data_); + // We just fumble ahead if we're unable to parse the AuthResponse. + } + LOG("network: authenticated viewfinder identity"); + + // TODO(peter): Passing AuthResponse::token_digits() in the error_id field + // is a hack. Yo! Clean this shit up. + done_(status_code, a.token_digits(), ""); + return AuthRequest::HandleDone(a); + } + + private: + const string endpoint_; + const string identity_; + const string password_; + const string first_; + const string last_; + const string name_; + const bool error_if_linked_; + const NetworkManager::AuthCallback done_; +}; + +class VerifyViewfinderRequest : public AuthRequest { + public: + VerifyViewfinderRequest(NetworkManager* net, const string& identity, + const string& access_token, bool manual_entry, + const NetworkManager::AuthCallback& done) + : AuthRequest(net), + identity_(identity), + access_token_(access_token), + manual_entry_(manual_entry), + done_(done) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + LOG("network: verify viewfinder, identity=\"%s\", access_token=\"%s\"", + identity_, access_token_); + JsonDict auth({ + { "identity", identity_ }, + { "access_token", access_token_ } }); + const string json = FormatRequest(auth, 0, true); + const string url = FormatUrl( + state(), Format("/%s/viewfinder", AppState::kVerifyEndpoint)); + NETLOG("network: verify_id: %s\n%s", url, json); + SendPost(url, json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + state()->analytics()->NetworkVerifyViewfinder(0, timer_.Get(), manual_entry_); + AuthRequest::HandleError(e); + done_(-1, ErrorResponse::UNKNOWN, e); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkVerifyViewfinder(status_code, timer_.Get(), manual_entry_); + + if (status_code != 200) { + LOG("network: verify viewfinder error: %d status: %s\n%s", + status_code, url(), data_); + // Note, unlike most other network requests, we do not retry on 5xx + // errors and instead pass back the error to the caller so an error can + // be displayed to the user. + ErrorResponse err; + if (!ParseErrorResponse(&err, data_)) { + done_(status_code, ErrorResponse::UNKNOWN, kDefaultVerifyErrorMessage); + } else { + done_(status_code, err.error().error_id(), err.error().text()); + } + return true; + } + + AuthResponse a; + if (!ParseAuthResponse(&a, data_)) { + LOG("network: unable to parse auth response: %s", data_); + return false; + } + + done_(status_code, ErrorResponse::OK, a.cookie()); + if (state()->registration_version() < AppState::REGISTRATION_EMAIL) { + state()->set_registration_version(AppState::current_registration_version()); + } + return AuthRequest::HandleDone(a); + } + + private: + const string identity_; + const string access_token_; + const bool manual_entry_; + const NetworkManager::AuthCallback done_; +}; + +class BenchmarkDownloadRequest : public NetworkRequest { + public: + BenchmarkDownloadRequest(NetworkManager* net, const string& url) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + url_(url), + total_bytes_(0) { + } + + ~BenchmarkDownloadRequest() { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + NETLOG("network: starting benchmark download: up=%d wifi=%d: %s", + net_->network_up(), net_->network_wifi(), url_); + SendGet(url_); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleData(const Slice& d) { + total_bytes_ += d.size(); + LOG("network: benchmark received %d bytes, %d bytes total: %s, %.03f ms", + d.size(), total_bytes_, url_, timer_.Milliseconds()); + } + + void HandleError(const string& e) { + LOG("network: benchmark download error: %s", e); + state()->analytics()->NetworkBenchmarkDownload(0, net_->network_up(), net_->network_wifi(), + url_, -1, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkBenchmarkDownload(status_code, net_->network_up(), net_->network_wifi(), + url_, total_bytes_, timer_.Get()); + + LOG("network: benchmark download finished with %d: %d bytes: %s: %.03f ms", + status_code, total_bytes_, url_, timer_.Milliseconds()); + return true; + } + + private: + string url_; + int64_t total_bytes_; +}; + +class DownloadPhotoRequest : public NetworkRequest { + public: + DownloadPhotoRequest(NetworkManager* net, const NetworkQueue::DownloadPhoto* d) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + download_(d), + path_(d->path), + url_(d->url), + delete_file_(true), + fd_(FileCreate(path_)) { + MD5_Init(&md5_ctx_); + CHECK_GE(fd_, 0) << "file descriptor is invalid"; + } + ~DownloadPhotoRequest() { + if (fd_ != -1) { + close(fd_); + fd_ = -1; + } + if (delete_file_) { + FileRemove(path_); + } + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::DownloadPhoto* d = download_; + if (url_.empty()) { + url_ = FormatUrl( + state(), Format("/episodes/%s/photos/%s%s", + d->episode->id().server_id(), + d->photo->id().server_id(), + PhotoURLSuffix(d->type))); + } + NETLOG("network: downloading photo: %s: %s", d->photo->id(), url_); + SendGet(url_); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleData(const Slice& d) { + MD5_Update(&md5_ctx_, d.data(), d.size()); + + // TODO(pmattis): Gracefully handle out-of-space errors. + const char* p = d.data(); + int n = d.size(); + while (n > 0) { + ssize_t res = write(fd_, p, n); + if (res < 0) { + LOG("write failed: %s: %d (%s)", path_, errno, strerror(errno)); + break; + } + p += res; + n -= res; + } + } + + void HandleError(const string& e) { + LOG("network: photo download error: %s", e); + state()->analytics()->NetworkDownloadPhoto(0, -1, PhotoTypeName(download_->type), timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkDownloadPhoto(status_code, status_code == 200 ? FileSize(path_) : -1, + PhotoTypeName(download_->type), timer_.Get()); + + if (fd_ != -1) { + close(fd_); + fd_ = -1; + } + + if (download_ == state()->net_queue()->queued_download_photo()) { + const NetworkQueue::DownloadPhoto* d = download_; + if (status_code == 403 && url_ == d->url) { + // Our download was forbidden and we were talking directly to s3. Tell + // the photo manager to retry. + LOG("network: photo download error: %d status (retrying): %s: %s", + status_code, d->photo->id(), url_); + state()->net_queue()->CommitQueuedDownloadPhoto(string(), true); + } else if (status_code != 200) { + // The photo doesn't exist. Mark it with an error. + LOG("network: photo download error: %d status (not-retrying): %s: %s", + status_code, d->photo->id(), url_); + state()->net_queue()->CommitQueuedDownloadPhoto(string(), false); + } else { + const string md5 = GetMD5(); + LOG("network: downloaded photo: %s: %d bytes: %s: %.03f ms", + d->photo->id(), FileSize(path_), url_, timer_.Milliseconds()); + state()->net_queue()->CommitQueuedDownloadPhoto(md5, false); + delete_file_ = false; + } + } + return true; + } + + string GetMD5() { + uint8_t digest[MD5_DIGEST_LENGTH]; + MD5_Final(&md5_ctx_, digest); + return BinaryToHex(Slice((const char*)digest, ARRAYSIZE(digest))); + } + + private: + const NetworkQueue::DownloadPhoto* const download_; + MD5_CTX md5_ctx_; + const string path_; + string url_; + bool delete_file_; + int fd_; +}; + +class FetchContactsRequest : public NetworkRequest { + public: + FetchContactsRequest(NetworkManager* net, + const NetworkManager::FetchContactsCallback& done) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + done_(done) { + } + + protected: + void SendFetch(const string& url) { + JsonDict auth("device", FormatDeviceDict(state())); + const string json = FormatRequest(auth); + NETLOG("network: fetch contacts: %s\n%s", url, json); + SendPost(url, json, kJsonContentType); + } + + void HandleError(const string& e) { + LOG("network: fetch contacts error: %s", e); + done_(""); + } + + bool HandleDone(int status_code) { + AuthResponse a; + if (!ParseAuthResponse(&a, data_)) { + LOG("network: unable to parse auth response: %s", data_); + return false; + } + LOG("network: initiated fetch contacts: %s", a.headers().op_id()); + done_(a.headers().op_id()); + return true; + } + + private: + const NetworkManager::FetchContactsCallback done_; +}; + +class FetchFacebookContactsRequest : public FetchContactsRequest { + public: + FetchFacebookContactsRequest(NetworkManager* net, + const NetworkManager::FetchContactsCallback& done, + const string& access_token) + : FetchContactsRequest(net, done), + access_token_(access_token) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + LOG("network: fetch facebook contacts"); + SendFetch(FormatUrl(state(), + Format("/%s/facebook?access_token=%s", + state()->kLinkEndpoint, + access_token_))); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + state()->analytics()->NetworkFetchFacebookContacts(0, timer_.Get()); + FetchContactsRequest::HandleError(e); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkFetchFacebookContacts(status_code, timer_.Get()); + return FetchContactsRequest::HandleDone(status_code); + } + + private: + const string access_token_; +}; + +class FetchGoogleContactsRequest : public FetchContactsRequest { + public: + FetchGoogleContactsRequest(NetworkManager* net, + const NetworkManager::FetchContactsCallback& done, + const string& refresh_token) + : FetchContactsRequest(net, done), + refresh_token_(refresh_token) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + LOG("network: fetch google contacts"); + SendFetch(FormatUrl(state(), + Format("/%s/google?refresh_token=%s", + state()->kLinkEndpoint, + refresh_token_))); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + state()->analytics()->NetworkFetchGoogleContacts(0, timer_.Get()); + FetchContactsRequest::HandleError(e); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkFetchGoogleContacts(status_code, timer_.Get()); + return FetchContactsRequest::HandleDone(status_code); + } + + private: + const string refresh_token_; +}; + +class MergeAccountsRequest : public NetworkRequest { + public: + MergeAccountsRequest(NetworkManager* net, + const string& identity, + const string& access_token, + const string& completion_db_key, + const NetworkManager::AuthCallback& done) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + identity_(identity), + access_token_(access_token), + completion_db_key_(completion_db_key), + op_id_(state()->NewLocalOperationId()), + done_(done) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + OpHeaders headers; + headers.set_op_id(op_id_); + headers.set_op_timestamp(WallTime_Now()); + JsonDict dict({ + { "source_identity", + JsonDict({ + { "identity", identity_ }, + { "access_token", access_token_ } }) }, + { "activity", FormatActivityDict( + state(), headers.op_id(), headers.op_timestamp()) } }); + + const string json = FormatRequest(dict, headers, state()); + NETLOG("network: merge accounts: %s", json); + SendPost(FormatUrl(state(), "/service/merge_accounts"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: merge accounts error: %s", e); + state()->analytics()->NetworkMergeAccounts(0, timer_.Get()); + if (done_) { + done_(-1, ErrorResponse::UNKNOWN, e); + } + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkMergeAccounts(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: merge account error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + ErrorResponse err; + if (!ParseErrorResponse(&err, data_)) { + done_(status_code, ErrorResponse::UNKNOWN, kDefaultChangePasswordErrorMessage); + } else { + done_(status_code, err.error().error_id(), err.error().text()); + } + return true; + } + + DBHandle updates = state()->NewDBTransaction(); + const string encoded_op_id = EncodeOperationId(state()->device_id(), op_id_); + state()->contact_manager()->ProcessMergeAccounts( + encoded_op_id, completion_db_key_, updates); + updates->Commit(); + + LOG("network: merge accounts: %s", encoded_op_id); + done_(status_code, ErrorResponse::OK, ""); + return true; + } + + private: + const string identity_; + const string access_token_; + const string completion_db_key_; + const int64_t op_id_; + const NetworkManager::AuthCallback done_; +}; + +class PingRequest : public NetworkRequest { + public: + PingRequest(NetworkManager* net) + : NetworkRequest(net, NETWORK_QUEUE_PING) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict ping("device", FormatDeviceDict(state())); + + const string json = FormatRequest(ping); + NETLOG("network: ping:\n%s", json); + SendPost(FormatUrl(state(), "/ping"), json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: ping error: %s", e); + state()->analytics()->NetworkPing(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkPing(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: ping error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + return true; + } + + PingResponse p; + if (!ParsePingResponse(&p, data_)) { + // Don't retry, we could be a really old version that can't handle the response. + // Reset the system message on bad responses. This ensures that we won't remain stuck + // in the DISABLE_NETWORK state. + LOG("network: unable to parse ping reponse"); + state()->clear_system_message(); + net_->SetNetworkDisallowed(false); + return true; + } + + if (!p.has_message()) { + state()->clear_system_message(); + VLOG("Got empty ping response"); + net_->SetNetworkDisallowed(false); + return true; + } + + // Set disallowed variable from here, no need to register a callback. + net_->SetNetworkDisallowed(p.message().severity() == SystemMessage::DISABLE_NETWORK); + + VLOG("Got ping response: %s", p); + state()->set_system_message(p.message()); + + return true; + } +}; + +class PostCommentRequest : public NetworkRequest { + public: + PostCommentRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d({ + { "viewpoint_id", u->viewpoint->id().server_id() }, + { "comment_id", u->comment->comment_id().server_id() }, + { "activity", FormatActivityDict(u->activity) } }); + if (!u->comment->asset_id().empty()) { + d.insert("asset_id", u->comment->asset_id()); + } + if (u->comment->has_timestamp()) { + d.insert("timestamp", u->comment->timestamp()); + } + if (!u->comment->message().empty()) { + d.insert("message", u->comment->message()); + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: post comment %s", u->activity->activity_id()); + SendPost(FormatUrl(state(), "/service/post_comment"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: post_comment error: %s", e); + state()->analytics()->NetworkPostComment(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkPostComment(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: post_comment error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + LOG("network: posted comment: %.03f", timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + } + return true; + } + + private: + const NetworkQueue::UploadActivity* const upload_; +}; + +class QueryContactsRequest : public NetworkRequest { + public: + QueryContactsRequest(NetworkManager* net, + const ContactSelection& contacts) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + contacts_(contacts) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d("limit", kQueryContactsLimit); + if (!contacts_.start_key().empty()) { + d.insert("start_key", contacts_.start_key()); + } + const string json = FormatRequest(d); + NETLOG("network: query contacts:\n%s", json); + SendPost(FormatUrl(state(), "/service/query_contacts"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query contacts error: %s", e); + state()->analytics()->NetworkQueryContacts(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryContacts(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: query contacts error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + QueryContactsResponse p; + if (!ParseQueryContactsResponse(&p, &contacts_, kQueryContactsLimit, data_)) { + LOG("network: unable to parse query_contacts response"); + return false; + } + + LOG("network: queried %d contact%s: %s: %d bytes, %.03f ms", + p.contacts_size(), Pluralize(p.contacts_size()), + p.last_key(), data_.size(), timer_.Milliseconds()); + + DBHandle updates = state()->NewDBTransaction(); + state()->contact_manager()->ProcessQueryContacts(p, contacts_, updates); + updates->Commit(); + return true; + } + + private: + ContactSelection contacts_; +}; + +class QueryEpisodesRequest : public NetworkRequest { + public: + QueryEpisodesRequest(NetworkManager* net, + const vector& episodes) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + episodes_(episodes), + limit_(std::max(1, kQueryObjectsLimit / episodes_.size())) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d({ + { "photo_limit", limit_ }, + { "episodes", + JsonArray(episodes_.size(), [&](int i) { + const EpisodeSelection& s = episodes_[i]; + JsonDict d({ + { "episode_id", s.episode_id() }, + { "get_attributes", s.get_attributes() }, + { "get_photos", s.get_photos() } + }); + if (s.has_get_photos() && !s.photo_start_key().empty()) { + d.insert("photo_start_key", s.photo_start_key()); + } + return d; + }) } + }); + const string json = FormatRequest(d); + NETLOG("network: query episodes:\n%s", json); + SendPost(FormatUrl(state(), "/service/query_episodes"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query episodes error: %s", e); + state()->analytics()->NetworkQueryEpisodes(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryEpisodes(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: query episodes error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + QueryEpisodesResponse p; + if (!ParseQueryEpisodesResponse( + &p, &episodes_, limit_, data_)) { + LOG("network: unable to parse query_episodes response"); + return false; + } + + int num_photos = 0; + for (int i = 0; i < p.episodes_size(); ++i) { + num_photos += p.episodes(i).photos_size(); + } + + LOG("network: queried %d episode%s, %d photo%s: %d bytes, %.03f ms", + p.episodes_size(), Pluralize(p.episodes_size()), + num_photos, Pluralize(num_photos), + data_.size(), timer_.Milliseconds()); + + DBHandle updates = state()->NewDBTransaction(); + state()->net_queue()->ProcessQueryEpisodes(p, episodes_, updates); + updates->Commit(); + return true; + } + + private: + vector episodes_; + const int limit_; +}; + +class QueryFollowedRequest : public NetworkRequest { + public: + QueryFollowedRequest(NetworkManager* net) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d("limit", kQueryFollowedLimit); + const string last_key = net_->query_followed_last_key_; + if (!last_key.empty()) { + d.insert("start_key", last_key); + } + const string json = FormatRequest(d); + NETLOG("network: query followed:\n%s", json); + SendPost(FormatUrl(state(), "/service/query_followed"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query followed error: %s", e); + state()->analytics()->NetworkQueryFollowed(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryFollowed(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: query followed error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + QueryFollowedResponse p; + if (!ParseQueryFollowedResponse(&p, data_)) { + LOG("network: unable to parse query_followed response"); + return false; + } + + LOG("network: query followed: %d viewpoint%s: %s: %.03f ms", + p.viewpoints_size(), Pluralize(p.viewpoints_size()), + p.last_key(), timer_.Milliseconds()); + + DBHandle updates = state()->NewDBTransaction(); + { + MutexLock l(&net_->mu_); + net_->need_query_followed_ = (p.viewpoints_size() >= kQueryFollowedLimit); + if (!net_->need_query_followed_) { + updates->Put(kQueryFollowedDoneKey, true); + // Force a query notification after we're done with the query_followed + // traversal. + state()->notification_manager()->Invalidate(updates); + } + if (p.has_last_key()) { + net_->query_followed_last_key_ = p.last_key(); + updates->Put(kQueryFollowedLastKey, net_->query_followed_last_key_); + } + } + state()->net_queue()->ProcessQueryFollowed(p, updates); + updates->Commit(); + return true; + } +}; + +class QueryNotificationsRequest : public NetworkRequest { + public: + QueryNotificationsRequest(NetworkManager* net, + const NotificationSelection& notifications, + bool long_poll) + : NetworkRequest(net, long_poll ? NETWORK_QUEUE_NOTIFICATION : NETWORK_QUEUE_REFRESH), + notifications_(notifications), + long_poll_(long_poll) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d; + if (net_->need_query_followed_) { + // We're performing a complete rebuild of our state, just retrieve the + // latest notification so that we have our notification high water mark. + d.insert("limit", 1); + d.insert("scan_forward", false); + notifications_.clear_last_key(); + } else { + d.insert("limit", kQueryNotificationsLimit); + } + if (!notifications_.last_key().empty()) { + d.insert("start_key", notifications_.last_key()); + } + if (long_poll_) { + d.insert("max_long_poll", kQueryNotificationsMaxLongPoll); + } + const string json = FormatRequest(d); + NETLOG("network: query notifications: %s", json); + SendPost(FormatUrl(state(), "/service/query_notifications"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query notifications error: %s", e); + state()->analytics()->NetworkQueryNotifications(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryNotifications(status_code, timer_.Get()); + + // Reset the application badge number to 0 to match server + // behavior--but only if application is active. + // TODO(spencer): Once the server supports a "clear_badge" flag, + // supply the value of clear_badge_ || app-active from here; + // server will always send ALL devices the badge=0 APNs alert. + if (net_->ShouldClearApplicationBadge()) { + NETLOG("network: clearing badge icon"); + net_->ClearApplicationBadge(); + } else { + NETLOG("network: not clearing badge icon"); + } + + if (status_code != 200) { + LOG("network: query notifications error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + QueryNotificationsResponse p; + if (!ParseQueryNotificationsResponse(&p, ¬ifications_, + kQueryNotificationsLimit, data_)) { + LOG("network: unable to parse query_notifications response"); + return false; + } + + LOG("network: queried %d notification%s (long poll: %d): %d bytes", + p.notifications_size(), Pluralize(p.notifications_size()), + long_poll_, data_.size()); + + // If the query returned non-empty results, reset background manager backoff. + if (p.notifications_size() > 0) { + net_->ResetQueryNotificationsBackoff(); + } + + DBHandle updates = state()->NewDBTransaction(); + // If we're querying all of our state, we're just trying to find + // the notification high-water mark and not actually processing + // any notifications. Pass in !need_query_followed_ to indicate to + // notification manager that it shouldn't call process callbacks. + state()->notification_manager()->ProcessQueryNotifications( + p, notifications_, !net_->need_query_followed_, updates); + if (net_->need_query_followed_) { + // If we're performing a query followed traversal we're not processing + // notifications. So clear any fetch contact operation that could be + // completed by a notification that we're skipping. + state()->contact_manager()->ClearFetchContacts(); + } + updates->Commit(); + + if (long_poll_ && p.retry_after() > 0) { + // If the server tells us to go away forever, don't listen. + const WallTime retry_after = std::min(p.retry_after(), kQueryNotificationsMaxRetryAfter); + LOG("network: pausing long polling for %s seconds", retry_after); + // 'this' will be deleted by the time the timeout fires, so copy member variables we need. + NetworkManager *const net = net_; + const NetworkManagerQueueType queue_type = queue_type_; + MutexLock lock(&net->mu_); + net->PauseLocked(queue_type); + dispatch_after_main(retry_after, [net, queue_type] { + MutexLock lock(&net->mu_); + net->ResumeLocked(queue_type); + }); + } + + return true; + } + + private: + NotificationSelection notifications_; + const bool long_poll_; +}; + +class QueryUsersRequest : public NetworkRequest { + public: + QueryUsersRequest(NetworkManager* net, const vector& user_ids) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + user_ids_(user_ids) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d("user_ids", + JsonArray(user_ids_.size(), [&](int i) { + return JsonValue(user_ids_[i]); + })); + const string json = FormatRequest(d); + NETLOG("network: query users:\n%s", json); + SendPost(FormatUrl(state(), "/service/query_users"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query users error: %s", e); + state()->analytics()->NetworkQueryUsers(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryUsers(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: query users error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + QueryUsersResponse p; + if (!ParseQueryUsersResponse(&p, data_)) { + LOG("network: unable to parse query_users response"); + return false; + } + + LOG("network: queried %d user%s: %d bytes, %.03f ms", + p.user_size(), Pluralize(p.user_size()), + data_.size(), timer_.Milliseconds()); + + DBHandle updates = state()->NewDBTransaction(); + state()->contact_manager()->ProcessQueryUsers(p, user_ids_, updates); + + // TODO(marc): The network manager is not a great place for this. Use the + // ContactManager::process_users() hook to place this elsewhere. + for (int i = 0; i < p.user_size(); ++i) { + const QueryUsersResponse::User& u = p.user(i); + if (u.contact().user_id() != state()->user_id()) { + continue; + } + + // User identities are processed by ContactManager.ProcessQueryUsers. + + // Handle account settings. + if (u.has_account_settings()) { + const AccountSettingsMetadata& a = u.account_settings(); + + // TODO: support email_alerts setting. + bool cloud_storage = false; + bool store_originals = false; + for (int j = 0; j < a.storage_options_size(); ++j) { + const string& option = a.storage_options(j); + if (option == "use_cloud") { + cloud_storage = true; + } else if (option == "store_originals") { + store_originals = true; + } + } + + LOG("Downloaded setting: cloud_storage=%d, store_originals=%d", + cloud_storage, store_originals); + state()->set_cloud_storage(cloud_storage); + state()->set_store_originals(store_originals); + + AppState* const s = state(); + s->async()->dispatch_main_async([s] { + // Note that "this" has been deleted at this point, so don't + // dereference it. + s->settings_changed()->Run(true); + }); + } + + // Handle no-password field. False if not present. + state()->set_no_password(u.no_password()); + break; + } + + updates->Commit(); + return true; + } + + private: + const vector user_ids_; +}; + +class QueryViewpointsRequest : public NetworkRequest { + public: + QueryViewpointsRequest(NetworkManager* net, + const vector& viewpoints) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + viewpoints_(viewpoints), + limit_(std::max(1, kQueryObjectsLimit / viewpoints_.size())) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d({ + { "limit", limit_ }, + { "viewpoints", + JsonArray(viewpoints_.size(), [&](int i) { + const ViewpointSelection& s = viewpoints_[i]; + JsonDict d("viewpoint_id", s.viewpoint_id()); + d.insert("get_activities", s.get_activities()); + d.insert("get_attributes", s.get_attributes()); + d.insert("get_episodes", s.get_episodes()); + d.insert("get_followers", s.get_followers()); + d.insert("get_comments", s.get_comments()); + if (s.has_get_activities() && !s.activity_start_key().empty()) { + d.insert("activity_start_key", s.activity_start_key()); + } + if (s.has_get_episodes() && !s.episode_start_key().empty()) { + d.insert("episode_start_key", s.episode_start_key()); + } + if (s.has_get_followers() && !s.follower_start_key().empty()) { + d.insert("follower_start_key", s.follower_start_key()); + } + if (s.has_get_comments() && !s.comment_start_key().empty()) { + d.insert("comment_start_key", s.comment_start_key()); + } + return d; + }) } }); + const string json = FormatRequest(d); + NETLOG("network: query viewpoints:\n%s", json); + SendPost(FormatUrl(state(), "/service/query_viewpoints"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: query viewpoints error: %s", e); + state()->analytics()->NetworkQueryViewpoints(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkQueryViewpoints(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: query viewpoints error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + QueryViewpointsResponse p; + if (!ParseQueryViewpointsResponse( + &p, &viewpoints_, limit_, data_)) { + LOG("network: unable to parse query_viewpoints response"); + return false; + } + + int num_episodes = 0; + for (int i = 0; i < p.viewpoints_size(); ++i) { + num_episodes += p.viewpoints(i).episodes_size(); + } + + LOG("network: queried %d viewpoint%s, %d episode%s: %d bytes, %.03f ms", + p.viewpoints_size(), Pluralize(p.viewpoints_size()), + num_episodes, Pluralize(num_episodes), + data_.size(), timer_.Milliseconds()); + + DBHandle updates = state()->NewDBTransaction(); + state()->net_queue()->ProcessQueryViewpoints(p, viewpoints_, updates); + updates->Commit(); + return true; + } + + private: + vector viewpoints_; + const int limit_; +}; + +class RecordSubscriptionRequest : public NetworkRequest { + public: + RecordSubscriptionRequest(NetworkManager* net, + const SubscriptionManager::RecordSubscription* r) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + record_(r) { + } + + void Start() { + net_->mu_.AssertHeld(); + JsonDict d("receipt_data", Base64Encode(record_->receipt_data)); + const string json = FormatRequest(d, record_->headers, state()); + SendPost(FormatUrl(state(), "/service/record_subscription"), + json, kJsonContentType); + } + + protected: + void HandleError(const string& e) { + LOG("network: record subscription error %s", e); + state()->analytics()->NetworkRecordSubscription(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkRecordSubscription(status_code, timer_.Get()); + ServerSubscriptionMetadata sub; + if (status_code != 200) { + LOG("network: record subscription error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } else { + const JsonValue d(ParseJSON(data_)); + if (!ParseServerSubscriptionMetadata(&sub, d["subscription"])) { + LOG("network: record subscription error: invalid subscription metadata: %s", data_); + return false; + } + } + + LOG("network: recorded subscription"); + DBHandle updates = state()->NewDBTransaction(); + // SubscriptionManager is currently iOS-specific, so + // RecordSubscriptionRequest is too. + state()->subscription_manager()->CommitQueuedRecordSubscription( + sub, status_code == 200, updates); + updates->Commit(); + return true; + } + + private: + const SubscriptionManager::RecordSubscription* record_; +}; + +class RemoveContactsRequest : public NetworkRequest { + public: + RemoveContactsRequest(NetworkManager* net, const ContactManager::RemoveContacts* remove) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + remove_(remove) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d("contacts", + JsonArray(remove_->server_contact_ids.size(), [&](int i) { + return JsonValue(remove_->server_contact_ids[i]); + })); + + const string json = FormatRequest(d, remove_->headers, state()); + NETLOG("network: remove contacts:\n%s", json); + SendPost(FormatUrl(state(), "/service/remove_contacts"), json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: remove contacts error: %s", e); + state()->analytics()->NetworkRemoveContacts(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkRemoveContacts(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: remove contacts error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } else { + LOG("network: removed contacts: %s: %.03f ms", + remove_->server_contact_ids.size(), timer_.Milliseconds()); + } + if (remove_ == state()->contact_manager()->queued_remove_contacts()) { + state()->contact_manager()->CommitQueuedRemoveContacts(status_code == 200); + } + return status_code == 200; + } + + private: + const ContactManager::RemoveContacts* remove_; +}; + +class RemoveFollowersRequest : public NetworkRequest { + public: + RemoveFollowersRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d({ + { "viewpoint_id", u->viewpoint->id().server_id() }, + { "activity", FormatActivityDict(u->activity) }, + { "remove_ids", + JsonArray(u->activity->remove_followers().user_ids_size(), [&](int i) { + return u->activity->remove_followers().user_ids(i); + }) } + }); + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: remove followers: %s", json); + SendPost(FormatUrl(state(), "/service/remove_followers"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: remove_followers error: %s", e); + state()->analytics()->NetworkRemoveFollowers(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkRemoveFollowers(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: remove_followers error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + const NetworkQueue::UploadActivity* u = upload_; + LOG("network: removed %d follower%s: %.03f", + u->activity->remove_followers().user_ids_size(), + Pluralize(u->activity->remove_followers().user_ids_size()), + timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + } + return true; + } + + private: + const NetworkQueue::UploadActivity* const upload_; +}; + +class RemovePhotosRequest : public NetworkRequest { + public: + RemovePhotosRequest(NetworkManager* net, const NetworkQueue::RemovePhotos* r) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + remove_(r) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::RemovePhotos* r = remove_; + + JsonDict d("episodes", + JsonArray(r->episodes.size(), [&](int i) { + const NetworkQueue::Episode& e = r->episodes[i]; + return JsonDict({ + { "episode_id", e.episode->id().server_id() }, + { "photo_ids", + JsonArray(e.photos.size(), [&](int j) { + return e.photos[j]->id().server_id(); + }) } + }); + })); + + const string json = FormatRequest(d, r->headers, state()); + NETLOG("network: remove_photos:\n%s", json); + SendPost(FormatUrl(state(), "/service/remove_photos"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + state()->analytics()->NetworkRemovePhotos(0, timer_.Get()); + LOG("network: remove photos error: %s", e); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkRemovePhotos(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: remove photos error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (remove_ == state()->net_queue()->queued_remove_photos()) { + if (status_code == 200) { + int num_photos = 0; + for (int i = 0; i < remove_->episodes.size(); ++i) { + num_photos += remove_->episodes[i].photos.size(); + } + LOG("network: removed %d photo%s from %d episode%s", + num_photos, Pluralize(num_photos), + remove_->episodes.size(), Pluralize(remove_->episodes.size())); + } + state()->net_queue()->CommitQueuedRemovePhotos(status_code != 200); + } + return true; + } + + private: + const NetworkQueue::RemovePhotos* const remove_; +}; + +class SavePhotosRequest : public NetworkRequest { + public: + SavePhotosRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + episode_count_(0), + photo_count_(0), + upload_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d("activity", FormatActivityDict(u->activity)); + if (!u->episodes.empty()) { + d.insert("episodes", + JsonArray(u->episodes.size(), [&](int i) { + const NetworkQueue::Episode& e = u->episodes[i]; + ++episode_count_; + photo_count_ += e.photos.size(); + return JsonDict({ + { "existing_episode_id", e.parent->id().server_id() }, + { "new_episode_id", e.episode->id().server_id() }, + { "photo_ids", + JsonArray(e.photos.size(), [&](int j) { + return e.photos[j]->id().server_id(); + }) } + }); + })); + } + // Viewpoint autosave photos. + if (u->activity->save_photos().has_viewpoint_id()) { + // TODO(spencer): enable this once support is on the server. + /* + d.insert("viewpoints", Array(1, [&](int i) { + return u->activity->save_photos().viewpoint_id().server_id(); + })); + */ + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: save photos: %s", json); + SendPost(FormatUrl(state(), "/service/save_photos"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: save_photos error: %s", e); + state()->analytics()->NetworkSavePhotos(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkSavePhotos(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: save_photos error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + LOG("network: saved %d photo%s from %d episode%s: %.03f", + photo_count_, Pluralize(photo_count_), + episode_count_, Pluralize(episode_count_), timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + } + return true; + } + + private: + int episode_count_; + int photo_count_; + const NetworkQueue::UploadActivity* const upload_; +}; + +class ShareRequest : public NetworkRequest { + public: + ShareRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u), + needs_invalidate_(false) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d; + if (u->activity->has_share_existing()) { + d.insert("viewpoint_id", u->viewpoint->id().server_id()); + } else { + JsonDict v({ + { "viewpoint_id", u->viewpoint->id().server_id() }, + { "type", "event" } + }); + if (!u->viewpoint->title().empty()) { + v.insert("title", u->viewpoint->title()); + } + if (u->viewpoint->has_cover_photo()) { + DCHECK(u->viewpoint->cover_photo().photo_id().has_server_id()); + DCHECK(u->viewpoint->cover_photo().episode_id().has_server_id()); + if (u->viewpoint->cover_photo().photo_id().has_server_id() && + u->viewpoint->cover_photo().episode_id().has_server_id()) { + const string photo_id = u->viewpoint->cover_photo().photo_id().server_id(); + const string episode_id = u->viewpoint->cover_photo().episode_id().server_id(); + // Only add the cover_photo field if the specified photo exists in + // the share request. + bool found = false; + for (int i = 0; !found && i < u->episodes.size(); ++i) { + const NetworkQueue::Episode& e = u->episodes[i]; + if (e.episode->id().server_id() == episode_id) { + for (int j = 0; !found && j < e.photos.size(); ++j) { + const PhotoHandle& p = e.photos[j]; + if (p->id().server_id() == photo_id) { + found = true; + v.insert("cover_photo", + JsonDict({ + { "photo_id", photo_id }, + { "episode_id", episode_id} + })); + break; + } + } + } + } + } + } + d.insert("viewpoint", v); + } + d.insert("activity", FormatActivityDict(u->activity)); + // The server requires "episodes" to be present for share_new even if it's empty. + if (!u->episodes.empty() || u->activity->has_share_new()) { + d.insert("episodes", + JsonArray(u->episodes.size(), [&](int i) { + const NetworkQueue::Episode& e = u->episodes[i]; + return JsonDict({ + { "existing_episode_id", e.parent->id().server_id() }, + { "new_episode_id", e.episode->id().server_id() }, + { "photo_ids", + JsonArray(e.photos.size(), [&](int j) { + return e.photos[j]->id().server_id(); + }) } + }); + })); + } + if (!u->contacts.empty()) { + d.insert("contacts", + JsonArray(u->contacts.size(), [&](int i) { + JsonDict d; + const ContactMetadata& c = u->contacts[i]; + if (c.has_primary_identity()) { + d.insert("identity", c.primary_identity()); + } + if (c.has_user_id()) { + d.insert("user_id", c.user_id()); + } else { + // If we upload followers without user ids (prospective users), they will have user ids + // assigned by this operation. The DayTable does not display followers that do not yet + // have user ids, so we need to fetch notifications once this is done. + needs_invalidate_ = true; + } + if (c.has_name()) { + d.insert("name", c.name()); + } + return d; + })); + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: share_%s:\n%s", + u->activity->has_share_existing() ? "existing" : "new", json); + SendPost(FormatUrl(state(), Format("/service/share_%s", + u->activity->has_share_existing() ? + "existing" : "new")), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: share error: %s", e); + state()->analytics()->NetworkShare(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkShare(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: share error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + const NetworkQueue::UploadActivity* u = upload_; + int num_photos = 0; + for (int i = 0; i < u->episodes.size(); ++i) { + num_photos += u->episodes[i].photos.size(); + } + LOG("network: shared %d episode%s, %d photo%s, %d contact%s", + u->episodes.size(), Pluralize(u->episodes.size()), + num_photos, Pluralize(num_photos), + u->contacts.size(), Pluralize(u->contacts.size())); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + + if (needs_invalidate_) { + AppState* const s = state(); + dispatch_after_main(kProspectiveUserCreationDelay, [s] { + DBHandle updates = s->NewDBTransaction(); + s->notification_manager()->Invalidate(updates); + updates->Commit(); + }); + } + } + return true; + } + + private: + const NetworkQueue::UploadActivity* const upload_; + bool needs_invalidate_; +}; + +class ResolveContactsRequest : public NetworkRequest { + public: + ResolveContactsRequest(NetworkManager* net, const std::string& identity) + : NetworkRequest(net, NETWORK_QUEUE_REFRESH), + identity_(identity) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + net_->mu_.AssertHeld(); + const string json = FormatRequest(JsonDict("identities", JsonArray({ identity_ }))); + SendPost(FormatUrl(state(), "/service/resolve_contacts"), + json, kJsonContentType); + } + + protected: + void HandleError(const string& e) { + LOG("network: resolve contacts error: %s", e); + state()->analytics()->NetworkResolveContacts(0, timer_.Get()); + state()->contact_manager()->ProcessResolveContact(identity_, NULL); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkResolveContacts(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: resolve contacts error: %d status: %s\n%s", + status_code, url(), data_); + state()->contact_manager()->ProcessResolveContact(identity_, NULL); + return false; + } + + ResolveContactsResponse resp; + if (!ParseResolveContactsResponse(&resp, data_)) { + LOG("network: unable to parse resolve_contacts response"); + state()->contact_manager()->ProcessResolveContact(identity_, NULL); + return false; + } + + if (resp.contacts_size() != 1 || + resp.contacts(0).primary_identity() != identity_) { + LOG("network: invalid resolve_contacts response"); + state()->contact_manager()->ProcessResolveContact(identity_, NULL); + return false; + } + + state()->contact_manager()->ProcessResolveContact(identity_, &resp.contacts(0)); + return true; + } + + private: + std::string identity_; +}; + +class UpdateFriendRequest : public NetworkRequest { + public: + UpdateFriendRequest(NetworkManager* net, int64_t user_id) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + user_id_(user_id) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + OpHeaders headers; + headers.set_op_id(state()->NewLocalOperationId()); + headers.set_op_timestamp(WallTime_Now()); + + ContactMetadata c; + CHECK(state()->contact_manager()->LookupUser(user_id_, &c)); + JsonDict friend_dict("user_id", user_id_); + if (c.nickname().empty()) { + friend_dict.insert("nickname", Json::Value::null); + } else { + friend_dict.insert("nickname", c.nickname()); + } + const JsonDict dict("friend", friend_dict); + + const string json = FormatRequest(dict, headers, state()); + NETLOG("network: update friend: %s", json); + SendPost(FormatUrl(state(), "/service/update_friend"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: update friend error: %s", e); + state()->analytics()->NetworkUpdateFriend(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUpdateFriend(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: update friend error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + LOG("network: updated friend metadata"); + state()->contact_manager()->CommitQueuedUpdateFriend(); + + return true; + } + + private: + const int64_t user_id_; +}; + +class UpdateUserRequest : public NetworkRequest { + public: + UpdateUserRequest(NetworkManager* net, const string& old_password, + const string& new_password, + const NetworkManager::AuthCallback& done) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + old_password_(old_password), + new_password_(new_password), + done_(done) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + OpHeaders headers; + headers.set_op_id(state()->NewLocalOperationId()); + headers.set_op_timestamp(WallTime_Now()); + + JsonDict dict("account_settings", FormatAccountSettingsDict(state())); + + ContactMetadata c; + if (state()->contact_manager()->LookupUser(state()->user_id(), &c)) { + // Note that we may not have a contact when the user initially logs in. + if (!c.name().empty()) { + dict.insert("name", c.name()); + } + if (!c.first_name().empty()) { + dict.insert("given_name", c.first_name()); + } + if (!c.last_name().empty()) { + dict.insert("family_name", c.last_name()); + } + } + + if (!old_password_.empty()) { + dict.insert("old_password", old_password_); + } + if (!new_password_.empty()) { + dict.insert("password", new_password_); + } + + const string json = FormatRequest(dict, headers, state()); + NETLOG("network: update user: %s", json); + SendPost(FormatUrl(state(), "/service/update_user"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: update user error: %s", e); + state()->analytics()->NetworkUpdateUser(0, timer_.Get()); + if (done_) { + done_(-1, ErrorResponse::UNKNOWN, e); + } + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUpdateUser(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: update user error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + if (done_) { + ErrorResponse err; + if (!ParseErrorResponse(&err, data_)) { + done_(status_code, ErrorResponse::UNKNOWN, kDefaultChangePasswordErrorMessage); + } else { + done_(status_code, err.error().error_id(), err.error().text()); + } + } + return true; + } + + LOG("network: updated user metadata"); + if (done_) { + done_(status_code, ErrorResponse::OK, ""); + } else { + state()->contact_manager()->CommitQueuedUpdateSelf(); + } + return true; + } + + private: + const string old_password_; + const string new_password_; + const NetworkManager::AuthCallback done_; +}; + +class UpdateUserPhotoRequest : public NetworkRequest { + public: + UpdateUserPhotoRequest(NetworkManager* net, const NetworkQueue::UpdatePhoto* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + update_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UpdatePhoto* u = update_; + JsonDict d("photo_id", u->photo->id().server_id()); + const JsonArray asset_keys(u->photo->asset_fingerprints_size(), [&](int i) { + return JsonValue(EncodeAssetKey("", u->photo->asset_fingerprints(i))); + }); + if (!asset_keys.empty()) { + d.insert("asset_keys", asset_keys); + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: update user photo:\n%s", json); + SendPost(FormatUrl(state(), "/service/update_user_photo"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: update user photo error: %s", e); + state()->analytics()->NetworkUpdateUserPhoto(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUpdateUserPhoto(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: update user photo error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (update_ == state()->net_queue()->queued_update_photo()) { + if (status_code == 200) { + LOG("network: updated user photo: %s: %.03f ms", + update_->photo->id(), timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUpdatePhoto(status_code != 200); + } + return true; + } + + private: + const NetworkQueue::UpdatePhoto* const update_; +}; + +class UpdateViewpointRequest : public NetworkRequest { + public: + UpdateViewpointRequest(NetworkManager* net, const NetworkQueue::UpdateViewpoint* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + update_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UpdateViewpoint* u = update_; + JsonDict d; + string service_endpoint; + + if (u->viewpoint->update_metadata()) { + service_endpoint = "/service/update_viewpoint"; + update_type_ = NetworkQueue::UPDATE_VIEWPOINT_METADATA; + d.insert("viewpoint_id", u->viewpoint->id().server_id()); + const string activity_id = EncodeActivityId( + state()->device_id(), u->headers.op_id(), u->headers.op_timestamp()); + d.insert("activity", + JsonDict({ + { "activity_id", activity_id }, + { "timestamp", u->headers.op_timestamp() } + })); + if (!u->viewpoint->title().empty()) { + d.insert("title", u->viewpoint->title()); + } + if (u->viewpoint->has_cover_photo()) { + d.insert("cover_photo", + JsonDict({ + { "photo_id", u->viewpoint->cover_photo().photo_id().server_id() }, + { "episode_id", u->viewpoint->cover_photo().episode_id().server_id() } + })); + } + if (!u->viewpoint->description().empty()) { + d.insert("description", u->viewpoint->description()); + } + if (!u->viewpoint->name().empty()) { + d.insert("name", u->viewpoint->name()); + } + } else if (u->viewpoint->update_remove()) { + service_endpoint = "/service/remove_viewpoint"; + update_type_ = NetworkQueue::UPDATE_VIEWPOINT_REMOVE; + d.insert("viewpoint_id", u->viewpoint->id().server_id()); + // Nothing else to set here. However, we must handle this case + // before we set labels, as it will be illegal to change the + // value of the "removed" label via a call to update follower + // metadata. + } else if (u->viewpoint->update_follower_metadata()) { + service_endpoint = "/service/update_follower"; + update_type_ = NetworkQueue::UPDATE_VIEWPOINT_FOLLOWER_METADATA; + vector labels; + if (u->viewpoint->label_admin()) { + labels.push_back("admin"); + } + if (u->viewpoint->label_autosave()) { + labels.push_back("autosave"); + } + if (u->viewpoint->label_contribute()) { + labels.push_back("contribute"); + } + if (u->viewpoint->label_hidden()) { + labels.push_back("hidden"); + } + if (u->viewpoint->label_muted()) { + labels.push_back("muted"); + } + if (u->viewpoint->label_removed()) { + labels.push_back("removed"); + } + // Only add labels if not empty; We should always have some + // permission, the exception being when the viewpoint was + // created locally and hasn't yet been uploaded. + if (!labels.empty()) { + JsonDict f("viewpoint_id", u->viewpoint->id().server_id()); + f.insert("labels", JsonArray(labels.size(), [&](int i) { + return labels[i]; + })); + d.insert("follower", f); + } + } else if (u->viewpoint->update_viewed_seq()) { + service_endpoint = "/service/update_follower"; + update_type_ = NetworkQueue::UPDATE_VIEWPOINT_VIEWED_SEQ; + JsonDict f("viewpoint_id", u->viewpoint->id().server_id()); + f.insert("viewed_seq", u->viewpoint->viewed_seq()); + d.insert("follower", f); + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: update viewpoint (type=%d):\n%s", update_type_, json); + SendPost(FormatUrl(state(), service_endpoint), json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: update viewpoint error: %s", e); + state()->analytics()->NetworkUpdateViewpoint(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUpdateViewpoint(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: update viewpoint error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (update_ == state()->net_queue()->queued_update_viewpoint()) { + if (status_code == 200) { + LOG("network: updated viewpoint (type=%d): %s: %.03f ms", + update_type_, update_->viewpoint->id(), timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUpdateViewpoint( + update_type_, status_code != 200); + } + return true; + } + + private: + const NetworkQueue::UpdateViewpoint* const update_; + NetworkQueue::UpdateViewpointType update_type_; +}; + +class UnshareRequest : public NetworkRequest { + public: + UnshareRequest(NetworkManager* net, const NetworkQueue::UploadActivity* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadActivity* u = upload_; + + JsonDict d({ + { "viewpoint_id", u->viewpoint->id().server_id() }, + { "activity", FormatActivityDict(u->activity) } + }); + if (!u->episodes.empty()) { + d.insert("episodes", + JsonArray(u->episodes.size(), [&](int i) { + const NetworkQueue::Episode& e = u->episodes[i]; + return JsonDict({ + { "episode_id", e.episode->id().server_id() }, + { "photo_ids", + JsonArray(e.photos.size(), [&](int j) { + return e.photos[j]->id().server_id(); + }) } + }); + })); + } + + const string json = FormatRequest(d, u->headers, state()); + NETLOG("network: unshare:\n%s", json); + SendPost(FormatUrl(state(), "/service/unshare"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: unshare error: %s", e); + state()->analytics()->NetworkUnshare(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUnshare(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: unshare error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_activity()) { + if (status_code == 200) { + const NetworkQueue::UploadActivity* u = upload_; + int num_photos = 0; + for (int i = 0; i < u->episodes.size(); ++i) { + num_photos += u->episodes[i].photos.size(); + } + LOG("network: unshared %d photo%s from %d episode%s", + num_photos, Pluralize(num_photos), + u->episodes.size(), Pluralize(u->episodes.size())); + } + state()->net_queue()->CommitQueuedUploadActivity(status_code != 200); + } + return true; + } + + private: + const NetworkQueue::UploadActivity* const upload_; +}; + +class UpdateDeviceRequest : public NetworkRequest { + public: + UpdateDeviceRequest(NetworkManager* net, bool* update_device) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + update_device_(update_device) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + OpHeaders headers; + headers.set_op_id(state()->NewLocalOperationId()); + headers.set_op_timestamp(WallTime_Now()); + + const string json = FormatRequest( + JsonDict("device_dict", FormatDeviceDict(state())), + headers, state()); + NETLOG("network: update device:\n%s", json); + SendPost(FormatUrl(state(), "/service/update_device"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: update device error: %s", e); + state()->analytics()->NetworkUpdateDevice(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUpdateDevice(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: update device error: %d status: %s\n%s", + status_code, url(), data_); + return false; + } + + LOG("network: updated device metadata"); + *update_device_ = false; + + return true; + } + + private: + bool* update_device_; +}; + +class UploadContactsRequest : public NetworkRequest { + public: + UploadContactsRequest(NetworkManager* net, const ContactManager::UploadContacts* upload) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(upload) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + JsonDict d("contacts", + JsonArray(upload_->contacts.size(), [&](int i) { + const ContactMetadata& m = upload_->contacts[i]; + JsonDict cd; +#define MAYBE_SET(proto_name, json_name) if (!m.proto_name().empty()) { cd.insert(json_name, m.proto_name()); } + MAYBE_SET(contact_source, "contact_source"); + MAYBE_SET(name, "name"); + MAYBE_SET(first_name, "given_name"); + MAYBE_SET(last_name, "family_name"); +#undef MAYBE_SET + if (m.has_rank()) { + cd.insert("rank", m.rank()); + } + cd.insert("identities", JsonArray(m.identities_size(), [&](int i) { + JsonDict ident("identity", m.identities(i).identity()); + if (m.identities(i).has_description()) { + ident.insert("description", m.identities(i).description()); + } + return ident; + })); + return cd; + })); + + const string json = FormatRequest(d, upload_->headers, state()); + NETLOG("network: upload contacts:\n%s", json); + SendPost(FormatUrl(state(), "/service/upload_contacts"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: upload contacts error: %s", e); + state()->analytics()->NetworkUploadContacts(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUploadContacts(status_code, timer_.Get()); + + UploadContactsResponse resp; + bool success = false; + if (status_code != 200) { + LOG("network: upload contacts error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx errors. + return false; + } + } else { + LOG("network: uploaded contacts: %s: %.03f ms", + upload_->contacts.size(), timer_.Milliseconds()); + + if (!ParseUploadContactsResponse(&resp, data_)) { + LOG("network: unable to parse upload contacts response"); + return false; + } + + success = true; + } + + if (upload_ == state()->contact_manager()->queued_upload_contacts()) { + state()->contact_manager()->CommitQueuedUploadContacts(resp, success); + } + + // Uploading contacts will generate silent notifications when server_contact_ids are assigned. + // Manually trigger another query_notifications to try and fetch them immediately. + DBHandle updates = state()->NewDBTransaction(); + state()->notification_manager()->Invalidate(updates); + updates->Commit(); + + return success; + } + + private: + const ContactManager::UploadContacts* upload_; +}; + +class UploadEpisodeRequest : public NetworkRequest { + public: + UploadEpisodeRequest(NetworkManager* net, const NetworkQueue::UploadEpisode* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadEpisode* u = upload_; + + JsonDict dict({ + { "episode", + JsonDict({ + { "timestamp", u->episode->timestamp() }, + { "episode_id", u->episode->id().server_id() } + }) }, + { "photos", + JsonArray(u->photos.size(), [&](int i) { + const PhotoMetadata& m = *u->photos[i]; + JsonDict d({ + { "timestamp", m.timestamp() }, + { "aspect_ratio", m.aspect_ratio() }, + { "content_type", ToString(kJpegContentType) }, + { "photo_id", m.id().server_id() } + }); + const JsonArray asset_keys(m.asset_fingerprints_size(), [&](int i) { + return JsonValue(EncodeAssetKey("", m.asset_fingerprints(i))); + }); + if (!asset_keys.empty()) { + d.insert("asset_keys", asset_keys); + } + if (m.has_images()) { + const PhotoMetadata::Images& images = m.images(); + if (images.has_tn()) { + d.insert("tn_size", images.tn().size()); + d.insert("tn_md5", images.tn().md5()); + } + if (images.has_med()) { + d.insert("med_size", images.med().size()); + d.insert("med_md5", images.med().md5()); + } + if (images.has_full()) { + d.insert("full_size", images.full().size()); + d.insert("full_md5", images.full().md5()); + } + if (images.has_orig()) { + d.insert("orig_size", images.orig().size()); + d.insert("orig_md5", images.orig().md5()); + } + } + if (m.has_location()) { + const Location& l = m.location(); + d.insert("location", + JsonDict({ + { "latitude", l.latitude() }, + { "longitude", l.longitude() } , + { "accuracy", l.accuracy() } + })); + if (m.has_placemark()) { + const Placemark& p = m.placemark(); + JsonDict t; + if (p.has_iso_country_code()) { + t.insert("iso_country_code", p.iso_country_code()); + } + if (p.has_country()) { + t.insert("country", p.country()); + } + if (p.has_state()) { + t.insert("state", p.state()); + } + if (p.has_locality()) { + t.insert("locality", p.locality()); + } + if (p.has_sublocality()) { + t.insert("sublocality", p.sublocality()); + } + if (p.has_thoroughfare()) { + t.insert("thoroughfare", p.thoroughfare()); + } + if (p.has_subthoroughfare()) { + t.insert("subthoroughfare", p.subthoroughfare()); + } + d.insert("placemark", t); + } + } + return d; + }) } + }); + dict.insert("activity", FormatActivityDict( + state(), u->headers.op_id(), u->headers.op_timestamp())); + + const string json = FormatRequest(dict, u->headers, state()); + NETLOG("network: upload episode\n%s", json); + SendPost(FormatUrl(state(), "/service/upload_episode"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: upload episode error: %s", e); + state()->analytics()->NetworkUploadEpisode(0, timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUploadEpisode(status_code, timer_.Get()); + + if (status_code != 200) { + LOG("network: upload episode error: %s status\n%s", + status_code, data_); + if ((status_code / 100) == 5) { + // Retry on 5xx status. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_episode()) { + UploadEpisodeResponse m; + if (status_code == 200) { + if (!ParseUploadEpisodeResponse(&m, data_)) { + LOG("network: unable to parse upload episode response"); + return false; + } + const NetworkQueue::UploadEpisode* u = upload_; + LOG("network: upload episode: %s: %d photo%s: %.03f ms", + u->episode->id(), u->photos.size(), Pluralize(u->photos.size()), + timer_.Milliseconds()); + } + state()->net_queue()->CommitQueuedUploadEpisode(m, status_code); + } + return true; + } + + private: + const NetworkQueue::UploadEpisode* const upload_; +}; + +class UploadLogToS3Request : public NetworkRequest { + public: + UploadLogToS3Request(NetworkManager* net, const string& url, + const string& path, const string& md5, + const std::shared_ptr& body) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + url_(url), + path_(path), + md5_(md5), + body_(body) { + } + + void Start() { + NETLOG("network: upload log to s3: %s: %s", url_, md5_); + SendPut(url_, *body_, kOctetStreamContentType, md5_, ""); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { +#ifdef PRODUCTION + LOG("network: upload log to s3 error: %s: %s", url(), e); +#else // !PRODUCTION + // In some non-production environments we can't upload the log to the + // server. Just delete. + FileRemove(path_); +#endif // !PRODUCTION + } + + bool HandleDone(int status_code) { + if (status_code != 200) { + LOG("network: upload log to s3 error: %d status: %s\n%s", + status_code, url(), data_); + if (((status_code / 100) == 5) || + IsS3RequestTimeout(status_code, data_)) { + // Retry on 5xx status and S3 timeouts. + return false; + } + } else { + LOG("network: uploaded log to s3: %s: %d bytes, %.03f ms", + path_, body_->size(), timer_.Milliseconds()); + FileRemove(path_); + } + return true; + } + + private: + const string url_; + const string path_; + const string md5_; + std::shared_ptr body_; +}; + +class UploadLogRequest : public NetworkRequest { + public: + UploadLogRequest(NetworkManager* net, const string& dir, const string& file) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + path_(JoinPath(dir, file)), + file_(file), + body_(new string) { + } + + void Start() { + // Extract the timestamp from the log filename. + WallTime timestamp; + string suffix; + if (!ParseLogFilename(file_, ×tamp, &suffix)) { + AsyncRemoveAndDelete(); + return; + } + + *body_ = ReadFileToString(path_); + if (body_->size() == 0) { + // Don't bother uploading 0 length logs. + AsyncRemoveAndDelete(); + return; + } + + md5_ = MD5HexToBase64(MD5(*body_)); + + const string client_log_id( + Format("%s%s", WallTimeFormat("%H-%M-%S", timestamp, false), suffix)); + const JsonDict d({ + { "client_log_id", client_log_id }, + { "timestamp", timestamp }, + { "content_type", ToString(kOctetStreamContentType) }, + { "content_md5", md5_ }, + { "num_bytes", static_cast(body_->size()) } + }); + + OpHeaders headers; + headers.set_op_id(state()->NewLocalOperationId()); + headers.set_op_timestamp(WallTime_Now()); + const string json = FormatRequest(d, headers, state()); + + NETLOG("network: upload log: %s: %s", file_, json); + SendPost(FormatUrl(state(), "/service/new_client_log_url"), + json, kJsonContentType); + } + + protected: + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: upload log error: %s", e); + } + + bool HandleDone(int status_code) { + if (status_code != 200) { + LOG("network: upload log error: %d status: %s\n%s", + status_code, url(), data_); + if ((status_code / 100) == 5) { + // Retry on 5xx status. + return false; + } + } else { + // The mutex must be held before invoking Start() on a new request. + MutexLock l(&net_->mu_); + const JsonValue d(ParseJSON(data_)); + const JsonRef url(d["client_log_put_url"]); + UploadLogToS3Request* req = new UploadLogToS3Request( + net_, url.string_value(), path_, md5_, body_); + req->Start(); + } + return true; + } + + private: + void AsyncRemoveAndDelete() { + state()->async()->dispatch_after_main(0, [this] { + FileRemove(path_); + net_->Dispatch(); + delete this; + }); + } + + private: + const string path_; + const string file_; + string md5_; + std::shared_ptr body_; +}; + +class UploadPhotoRequest : public NetworkRequest { + public: + UploadPhotoRequest(NetworkManager* net, const NetworkQueue::UploadPhoto* u) + : NetworkRequest(net, NETWORK_QUEUE_SYNC), + upload_(u), + photo_id_(u->photo->id().local_id()), + path_(u->path), + md5_(u->md5), + md5_base64_(MD5HexToBase64(md5_)) { + } + + // Start is called with NetworkManager::mu_ held. + void Start() { + const NetworkQueue::UploadPhoto* u = upload_; + string url = u->url; + string body; + string if_none_match; + + if (url.empty()) { + url = FormatUrl( + state(), Format("/episodes/%s/photos/%s%s", + u->episode->id().server_id(), + u->photo->id().server_id(), + PhotoURLSuffix(u->type))); + if_none_match = Format("\"%s\"", md5_); + } else { + body = ReadFileToString(path_); + } + + NETLOG("network: upload photo: %s: %s: %s", u->photo->id(), url, md5_); + SendPut(url, body, kJpegContentType, md5_base64_, if_none_match); + } + + protected: + void HandleRedirect( + ScopedPtr* new_body, + StringSet* delete_headers, StringMap* add_headers) { + // Add in the body now that we've been redirected to s3. + new_body->reset(new string); + **new_body = ReadFileToString(path_); + // Strip out the If-None-Match header which will cause s3 to fail. + delete_headers->insert("If-None-Match"); + (*add_headers)["Content-Type"] = kJpegContentType; + (*add_headers)["Content-MD5"] = md5_base64_; + } + + // The various Handle*() methods are called on the NetworkManager + // thread. + void HandleError(const string& e) { + LOG("network: upload photo error: %s", e); + state()->analytics()->NetworkUploadPhoto(0, -1, PhotoTypeName(upload_->type), timer_.Get()); + } + + bool HandleDone(int status_code) { + state()->analytics()->NetworkUploadPhoto( + status_code, status_code == 200 ? FileSize(path_) : -1, + PhotoTypeName(upload_->type), timer_.Get()); + + bool error = (status_code != 200 && status_code != 304); + if (error) { + LOG("network: upload photo error: %d status: %s\n%s", + status_code, url(), data_); + if (((status_code / 100) == 5) || + IsS3RequestTimeout(status_code, data_)) { + // Retry on 5xx status and S3 timeouts. + return false; + } + } + if (upload_ == state()->net_queue()->queued_upload_photo()) { + const NetworkQueue::UploadPhoto* u = upload_; + LOG("network: upload photo (%d): %s: %d bytes: %s: %.03f ms", + status_code, u->photo->id(), FileSize(path_), + md5_, timer_.Milliseconds()); + // Use VLOG for the headers to minimize spamming the debug console. + VLOG("network: upload photo: %s", u->photo->id()); + state()->net_queue()->CommitQueuedUploadPhoto(error); + } + return true; + } + + private: + const NetworkQueue::UploadPhoto* const upload_; + const int64_t photo_id_; + const string path_; + const string md5_; + const string md5_base64_; +}; + +NetworkRequest::NetworkRequest(NetworkManager* net, NetworkManagerQueueType queue) + : net_(net), + state_(net_->state()), + queue_type_(queue) { +} + +NetworkRequest::~NetworkRequest() { +} + +void NetworkRequest::HandleRedirect( + ScopedPtr* new_body, + StringSet* delete_headers, StringMap* add_headers) { +} + +void NetworkRequest::HandleData(const Slice& d) { + data_.append(d.data(), d.size()); +} + +void NetworkRequest::SendGet(const string& url) { + Send(url, "GET", "", "", "", ""); +} + +void NetworkRequest::SendPost( + const string& url, const Slice& body, const Slice& content_type) { + Send(url, "POST", body, content_type, "", ""); +} + +void NetworkRequest::SendPut( + const string& url, const Slice& body, + const Slice& content_type, const Slice& content_md5, + const Slice& if_none_match) { + Send(url, "PUT", body, content_type, content_md5, if_none_match); +} + +void NetworkRequest::Send( + const string& url, const Slice& method, const Slice& body, + const Slice& content_type, const Slice& content_md5, + const Slice& if_none_match) { + url_ = url; + net_->SendRequest(this, method, body, content_type, content_md5, if_none_match); +} + +NetworkManager::NetworkManager(AppState* state) + : state_(state), + pause_non_interactive_count_(0), + epoch_(0), + last_request_success_(true), + refreshing_(false), + network_reachable_(false), + network_wifi_(false), + need_query_followed_(false), + update_device_(false), + assets_scanned_(false), + draining_(false), + network_disallowed_(false), + register_new_user_(false), + last_ping_timestamp_(0), + fake_401_(false) { + if (state_->server_host().empty()) { + // Networking is disabled. + return; + } + + query_followed_last_key_ = + state_->db()->Get(kQueryFollowedLastKey); + + state_->notification_manager()->nuclear_invalidations()->Add( + [this](const DBHandle& updates) { + NuclearInvalidation(updates); + }); +} + +NetworkManager::~NetworkManager() { + for (int i = 0; i < NUM_NETWORK_QUEUE_TYPES; i++) { + CHECK_EQ(queue_state_[i].network_count, 0); + } +} + +void NetworkManager::Dispatch() { + // NSURLConnection objects are unhappy unless they are created on the main + // thread. + if (!dispatch_is_main_thread()) { + NETLOG("network dispatch: called on non-main thread"); + return; + } + MutexLock l(&mu_); + if (draining_) { + NETLOG("network dispatch: draining"); + return; + } + + // If we're experiencing errors and don't know whether the network is up, don't do anything. + // As long as we're not experiencing errors, go ahead and try even if the reachability + // check hasn't told us the network is up yet. + if (!network_up()) { + if (refreshing_) { + refreshing_ = false; + refresh_end_.Run(); + } + return; + } + + int orig_network_count[NUM_NETWORK_QUEUE_TYPES]; + for (int i = 0; i < NUM_NETWORK_QUEUE_TYPES; i++) { + orig_network_count[i] = queue_state_[i].network_count; + } + + // Dispatch the subqueues and verify that they don't try to start any operations for the other queues. + for (int i = 0; i < NUM_NETWORK_QUEUE_TYPES; i++) { + if (queue_is_busy(static_cast(i))) { + continue; + } + if (i != NETWORK_QUEUE_PING) { + if (network_disallowed_) { + // Network is temporarily disallowed due to system message. + // Only Ping can go through (only way to re-allow). + NETLOG("network dispatch: network disallowed"); + break; + } + if (queue_state_[NETWORK_QUEUE_PING].network_count > 0) { + // While a ping is in progress, block all the other queues. + break; + } + } + switch (i) { + case NETWORK_QUEUE_PING: + DispatchPingLocked(); + break; + case NETWORK_QUEUE_REFRESH: + DispatchRefreshLocked(); + break; + case NETWORK_QUEUE_NOTIFICATION: + DispatchNotificationLocked(); + break; + case NETWORK_QUEUE_SYNC: + DispatchSyncLocked(); + break; + default: + DIE("unknown queue type %s", i); + } + orig_network_count[i] = queue_state_[i].network_count; + for (int j = 0; j < NUM_NETWORK_QUEUE_TYPES; j++) { + DCHECK_EQ(orig_network_count[j], queue_state_[j].network_count); + } + } + + SetIdleTimer(); +} + +void NetworkManager::DispatchPingLocked() { + mu_.AssertHeld(); + + MaybePingServer(); +} + +void NetworkManager::DispatchRefreshLocked() { + mu_.AssertHeld(); + + // The rest of the requests require authentication. + if (need_auth()) { + if (refreshing_) { + refreshing_ = false; + refresh_end_.Run(); + } + NETLOG("network dispatch: need auth"); + return; + } + + if (pause_non_interactive()) { + NETLOG("network dispatch: pause non-interactive: %d", + pause_non_interactive_count_); + return; + } + + const QueueState& queue_state = queue_state_[NETWORK_QUEUE_REFRESH]; + + // Query for notifications and revalidate invalidated data. Only + // perform this revalidation work the first time through the loop as it + // is unexpected for it to change on a subsequent pass. + do { + MaybeQueryNotifications(false); if (queue_state.network_count) break; + MaybeQueryUsers(); if (queue_state.network_count) break; + if (assets_scanned_) { + // Only query for photos/episodes/viewpoints after we've finished the + // first asset scan as we want to ensure we can match up local photos + // with those returned from the server so that we can avoid + // downloading the photos from the server if we have the photo + // locally. + // + // TODO(peter): We could lift this restriction if a photo found + // during the asset scan could clear the "download_*" bits of the + // PhotoMetadata. + MaybeQueryEpisodes(); if (queue_state.network_count) break; + MaybeQueryViewpoints(); if (queue_state.network_count) break; + MaybeQueryFollowed(); if (queue_state.network_count) break; + } + MaybeQueryContacts(); if (queue_state.network_count) break; + } while (0); + + if (queue_state.network_count) { + if (!refreshing_) { + refreshing_ = true; + refresh_start_.Run(); + } + return; + } + + if (refreshing_) { + refreshing_ = false; + if (assets_scanned_) { + // Only mark the refresh as completed if we were actually able to query for everything. + state_->set_refresh_completed(true); + } + refresh_end_.Run(); + } +} + +void NetworkManager::DispatchNotificationLocked() { + mu_.AssertHeld(); + + // Let any refreshes finish before going into a long poll. The reverse is not true; + // a refresh may start a short request while a long poll is in progress. This is slightly less + // efficient and we may want to change it as we gain more confidence in our long-polling system, + // but for now this ensures that the app doesn't become unresponsive if a long poll becomes stuck + // in some way that is slow to report an error. + if (need_auth() || refreshing_) { + return; + } + + MaybeQueryNotifications(true); +} + +void NetworkManager::DispatchSyncLocked() { + mu_.AssertHeld(); + + if (need_auth()) { + return; + } + + const QueueState& queue_state = queue_state_[NETWORK_QUEUE_SYNC]; + + // 1. Upload logs if the last request was not successful. Otherwise we'll + // upload logs at the end of this queue as the lowest priority operation. + if (!pause_non_interactive() && !last_request_success_) { + MaybeUploadLog(); if (queue_state.network_count) return; + } + + // 2. Prioritize sending update device, update friend/user, link identity, + // and subscription requests. + if (!pause_non_interactive()) { + MaybeUpdateDevice(); if (queue_state.network_count) return; + MaybeUpdateFriend(); if (queue_state.network_count) return; + MaybeUpdateUser(); if (queue_state.network_count) return; + MaybeRecordSubscription(); if (queue_state.network_count) return; + } + + // Don't do anything photo-related until the initial asset scan has run (so we have asset + // fingerprints to match against) + if (!assets_scanned_) { + return; + } + + // 3. Indicate the network is ready so that photo downloads and other + // operations will be queued. + if (pause_non_interactive()) { + // If non-interactive operations are paused, only queue thumbnail and + // full photo downloads. + state_->network_ready()->Run(PRIORITY_UI_FULL); + } else { + state_->network_ready()->Run(PRIORITY_UI_MAX); + } + + // 4. Download photos before querying for notifications as the UI might have + // queued up the download requests (e.g. because of a non-existent or corrupt + // image). + MaybeDownloadPhoto(); if (queue_state.network_count) return; + + // 5. Other photo operations. + if (!pause_non_interactive()) { + state_->network_ready()->Run(PRIORITY_MAX); + } + // Note that we let any already queued photo manager operations start and + // complete even if pause_non_interactive is true so that we can queue new + // download photo operations. + NETLOG("network dispatch: running all priorities"); + MaybeDownloadPhoto(); if (queue_state.network_count) return; + MaybeUploadPhoto(); if (queue_state.network_count) return; + MaybeRemovePhotos(); if (queue_state.network_count) return; + MaybeUploadEpisode(); if (queue_state.network_count) return; + MaybeUploadActivity(); if (queue_state.network_count) return; + MaybeUpdatePhoto(); if (queue_state.network_count) return; + MaybeUpdateViewpoint(); if (queue_state.network_count) return; + MaybeUploadContacts(); if (queue_state.network_count) return; + MaybeRemoveContacts(); if (queue_state.network_count) return; + + if (!pause_non_interactive()) { + // 6. Logs. + MaybeUploadLog(); if (queue_state.network_count) return; + } + +#ifdef DEVELOPMENT + MaybeBenchmarkDownload(); if (queue_state.network_count) return; +#endif // DEVELOPMENT +} + +bool NetworkManager::Refresh() { + // Force a query notification. + DBHandle updates = state_->NewDBTransaction(); + state_->notification_manager()->Invalidate(updates); + updates->Commit(); + + if (!assets_scanned_) { + return false; + } + + refreshing_ = true; + refresh_start_.Run(); + + Dispatch(); + return true; +} + +void NetworkManager::ResetBackoff() { + MutexLock lock(&mu_); + + last_request_success_ = true; + for (int i = 0; i < NUM_NETWORK_QUEUE_TYPES; i++) { + QueueState* state = &queue_state_[i]; + state->backoff_delay = kMinBackoffDelay; + if (state->backoff_count) { + NETLOG("network: reset backoff %s: %d", i, state->backoff_count); + ResumeFromBackoffLocked(static_cast(i)); + } + } +} + +void NetworkManager::ResolveContact(const string& identity) { + if (need_auth()) { + state_->contact_manager()->ProcessResolveContact(identity, NULL); + return; + } + + MutexLock lock(&mu_); + + ResolveContactsRequest* req = new ResolveContactsRequest(this, identity); + req->Start(); +} + +bool NetworkManager::FetchFacebookContacts( + const string& access_token, const FetchContactsCallback& done) { + CHECK(dispatch_is_main_thread()); + + MutexLock l(&mu_); + + if (draining_ || need_auth()) { + return false; + } + + // Immediately kick off the fetch contacts request. + FetchFacebookContactsRequest* req = + new FetchFacebookContactsRequest( + this, done, access_token); + req->Start(); + return true; +} + +bool NetworkManager::FetchGoogleContacts( + const string& refresh_token, const FetchContactsCallback& done) { + CHECK(dispatch_is_main_thread()); + + MutexLock l(&mu_); + + if (draining_ || need_auth()) { + return false; + } + + // Immediately kick off the fetch contacts request. + FetchGoogleContactsRequest* req = + new FetchGoogleContactsRequest( + this, done, refresh_token); + req->Start(); + return true; +} + +void NetworkManager::AuthViewfinder( + const string& endpoint, const string& identity, const string& password, + const string& first, const string& last, const string& name, + bool error_if_linked, const AuthCallback& done) { + CHECK(dispatch_is_main_thread()); + MutexLock l(&mu_); + + if (draining_ || xsrf_cookie().empty()) { + ResetPing(); + dispatch_after_main(0, [done] { + done(-1, ErrorResponse::NETWORK_UNAVAILABLE, kDefaultNetworkErrorMessage); + }); + return; + } + + // We need to keep track of whether we are registering a new user or doing + // some other auth request (such as login or password reset). We'll use this + // state when AuthDone() is called to determine whether we have to wait for + // the initial asset scan to finish before querying server state or not. + register_new_user_ = (endpoint == AppState::kRegisterEndpoint); + + // Immediately kick off the auth request. + AuthViewfinderRequest* req = new AuthViewfinderRequest( + this, endpoint, identity, password, first, + last, name, error_if_linked, done); + req->Start(); +} + +void NetworkManager::VerifyViewfinder( + const string& identity, const string& access_token, + bool manual_entry, const AuthCallback& done) { + CHECK(dispatch_is_main_thread()); + MutexLock l(&mu_); + + if (draining_ || xsrf_cookie().empty()) { + ResetPing(); + dispatch_after_main(0, [done] { + done(-1, ErrorResponse::NETWORK_UNAVAILABLE, kDefaultNetworkErrorMessage); + }); + return; + } + + // Immediately kick off the auth request. + VerifyViewfinderRequest* req = new VerifyViewfinderRequest( + this, identity, access_token, manual_entry, done); + req->Start(); +} + +void NetworkManager::ChangePassword( + const string& old_password, const string& new_password, + const AuthCallback& done) { + // TODO(peter): Spencer notes there is a bunch of shared code between this + // method, AuthViewfinder and VerifyViewfinder. + CHECK(dispatch_is_main_thread()); + MutexLock l(&mu_); + + if (draining_ || xsrf_cookie().empty()) { + ResetPing(); + dispatch_after_main(0, [done] { + done(-1, ErrorResponse::NETWORK_UNAVAILABLE, kDefaultNetworkErrorMessage); + }); + return; + } + + // Immediately kick off the change password request. + UpdateUserRequest* req = new UpdateUserRequest( + this, old_password, new_password, done); + req->Start(); +} + +void NetworkManager::MergeAccounts( + const string& identity, const string& access_token, + const string& completion_db_key, + const AuthCallback& done) { + // TODO(peter): Spencer notes there is a bunch of shared code between this + // method, AuthViewfinder and VerifyViewfinder. + CHECK(dispatch_is_main_thread()); + MutexLock l(&mu_); + + if (draining_ || xsrf_cookie().empty()) { + ResetPing(); + dispatch_after_main(0, [done] { + done(-1, ErrorResponse::NETWORK_UNAVAILABLE, kDefaultNetworkErrorMessage); + }); + return; + } + + // Immediately kick off the merge accounts request. + MergeAccountsRequest* req = new MergeAccountsRequest( + this, identity, access_token, completion_db_key, done); + req->Start(); +} + +void NetworkManager::SetPushNotificationDeviceToken(const string& base64_token) { + LOG("network: push notification device token: %s", base64_token); + state_->db()->Put(kPushDeviceTokenKey, base64_token); + + update_device_ = true; + Dispatch(); +} + +void NetworkManager::PauseNonInteractive() { + MutexLock l(&mu_); + ++pause_non_interactive_count_; + NETLOG("network: pause non-interactive: %d", pause_non_interactive_count_); +} + +void NetworkManager::ResumeNonInteractive() { + MutexLock l(&mu_); + --pause_non_interactive_count_; + NETLOG("network: resume non-interactive: %d", pause_non_interactive_count_); + + // Note that we intentionally do not call dispatch_main() here as we want the + // stack to unwind and locks to be released before Dispatch() is called. + state_->async()->dispatch_main_async([this] { + Dispatch(); + }); +} + +void NetworkManager::SetNetworkDisallowed(bool disallow) { + if (disallow == network_disallowed_) { + return; + } + // Full system message is logged by AppState, only log network status switch. + LOG("Setting network_disallowed=%s", disallow ? "true" : "false"); + network_disallowed_ = disallow; +} + +void NetworkManager::RunDownloadBenchmark() { + MutexLock l(&mu_); + // Don't clear the queue, we want to finish the previous benchmark, if any. + for (int i = 0; i < ARRAYSIZE(kDownloadBenchmarkFiles); ++i) { + benchmark_urls_.push_back(kDownloadBenchmarkURLPrefix + kDownloadBenchmarkFiles[i]); + } + + // Trigger a Dispatch to start right away. + state_->async()->dispatch_main_async([this] { + Dispatch(); + }); +} + +void NetworkManager::Logout(bool clear_user_id) { + MutexLock l(&mu_); + state_->SetAuthCookies(string(), string()); + // Force the app back into the signup/login state. + state_->SetUserAndDeviceId(clear_user_id ? 0 : state_->user_id(), 0); +} + +void NetworkManager::UnlinkDevice() { + MutexLock l(&mu_); + ++epoch_; + for (int i = 0; i < NUM_NETWORK_QUEUE_TYPES; i++) { + queue_state_[i].backoff_delay = kMinBackoffDelay; + } + need_query_followed_ = true; + query_followed_last_key_.clear(); + assets_scanned_ = false; + refreshing_ = false; + refresh_end_.Run(); +} + +void NetworkManager::Drain() { + MutexLock l(&mu_); + draining_ = true; +} + +void NetworkManager::AssetScanEnd() { + state_->async()->dispatch_main([this]{ + assets_scanned_ = true; + need_query_followed_ = !state_->db()->Exists(kQueryFollowedDoneKey); + if (need_query_followed_ && + !state_->db()->Exists(kQueryFollowedLastKey)) { + LOG("network: initial state rebuild, synthesizing nuclear invalidate"); + // The query followed done key does not exist, synthesize a nuclear + // invalidation. + DBHandle updates = state_->NewDBTransaction(); + state_->notification_manager()->nuclear_invalidations()->Run(updates); + updates->Commit(); + } + Dispatch(); + }); +} + +void NetworkManager::MaybePingServer() { + WallTime time_since_last_ping = WallTime_Now() - last_ping_timestamp_; + if (time_since_last_ping > kPingPeriodDefault || + (network_disallowed_ && time_since_last_ping > kPingPeriodFast)) { + last_ping_timestamp_ = WallTime_Now(); + PingRequest* req = new PingRequest(this); + req->Start(); + } +} + +void NetworkManager::MaybeBenchmarkDownload() { + if (benchmark_urls_.empty()) { + return; + } + + BenchmarkDownloadRequest* req = new BenchmarkDownloadRequest(this, benchmark_urls_.front()); + benchmark_urls_.pop_front(); + req->Start(); +} + +void NetworkManager::MaybeDownloadPhoto() { + const NetworkQueue::DownloadPhoto* d = + state_->net_queue()->queued_download_photo(); + if (!d) { + return; + } + DownloadPhotoRequest* req = new DownloadPhotoRequest(this, d); + req->Start(); +} + +void NetworkManager::MaybeQueryContacts() { + ContactSelection contacts; + if (!state_->contact_manager()->GetInvalidation(&contacts)) { + return; + } + QueryContactsRequest* req = new QueryContactsRequest(this, contacts); + req->Start(); +} + +void NetworkManager::MaybeQueryEpisodes() { + vector episodes; + state_->episode_table()->ListInvalidations( + &episodes, kQueryEpisodesLimit, state_->db()); + if (episodes.empty()) { + return; + } + QueryEpisodesRequest* req = new QueryEpisodesRequest(this, episodes); + req->Start(); +} + +void NetworkManager::MaybeQueryFollowed() { + if (!need_query_followed_ || !assets_scanned_) { + return; + } + QueryFollowedRequest* req = new QueryFollowedRequest(this); + req->Start(); +} + +void NetworkManager::MaybeQueryNotifications(bool long_poll) { + NotificationSelection notifications; + if (!assets_scanned_) { + return; + } + if (need_query_followed_ && !query_followed_last_key_.empty()) { + // If we're rebuilding our list of viewpoints, let that finish before querying for more notifications. + return; + } + const bool need_query = state_->notification_manager()->GetInvalidation(¬ifications); + if (!need_query && !long_poll) { + // If we think we're up to date, don't query unless we're in long-poll mode. + return; + } + QueryNotificationsRequest* req = new QueryNotificationsRequest(this, notifications, long_poll); + req->Start(); +} + +void NetworkManager::MaybeQueryUsers() { + vector user_ids; + state_->contact_manager()->ListQueryUsers(&user_ids, kQueryUsersLimit); + if (user_ids.empty()) { + return; + } + QueryUsersRequest* req = new QueryUsersRequest(this, user_ids); + req->Start(); +} + +void NetworkManager::MaybeQueryViewpoints() { + vector viewpoints; + state_->viewpoint_table()->ListInvalidations( + &viewpoints, kQueryViewpointsLimit, state_->db()); + if (viewpoints.empty()) { + return; + } + QueryViewpointsRequest* req = new QueryViewpointsRequest(this, viewpoints); + req->Start(); +} + +void NetworkManager::MaybeRecordSubscription() { + if (!state_->subscription_manager()) { + return; + } + const SubscriptionManager::RecordSubscription* r = + state_->subscription_manager()->GetQueuedRecordSubscription(); + if (!r) { + return; + } + RecordSubscriptionRequest* req = new RecordSubscriptionRequest(this, r); + req->Start(); +} + +void NetworkManager::MaybeRemovePhotos() { + const NetworkQueue::RemovePhotos* r = + state_->net_queue()->queued_remove_photos(); + if (!r) { + return; + } + RemovePhotosRequest* req = new RemovePhotosRequest(this, r); + req->Start(); +} + +void NetworkManager::MaybeUpdateDevice() { + if (!update_device_) { + return; + } + UpdateDeviceRequest* req = new UpdateDeviceRequest(this, &update_device_); + req->Start(); +} + +void NetworkManager::MaybeUpdateFriend() { + if (!state_->contact_manager()->queued_update_friend()) { + return; + } + UpdateFriendRequest* req = new UpdateFriendRequest( + this, state_->contact_manager()->queued_update_friend()); + req->Start(); +} + +void NetworkManager::MaybeUpdatePhoto() { + const NetworkQueue::UpdatePhoto* u = + state_->net_queue()->queued_update_photo(); + if (!u) { + return; + } + UpdateUserPhotoRequest* req = new UpdateUserPhotoRequest(this, u); + req->Start(); +} + +void NetworkManager::MaybeUpdateUser() { + if (!state_->contact_manager()->queued_update_self()) { + return; + } + UpdateUserRequest* req = new UpdateUserRequest(this, "", "", NULL); + req->Start(); +} + +void NetworkManager::MaybeUpdateViewpoint() { + const NetworkQueue::UpdateViewpoint* u = + state_->net_queue()->queued_update_viewpoint(); + if (!u) { + return; + } + UpdateViewpointRequest* req = new UpdateViewpointRequest(this, u); + req->Start(); +} + +void NetworkManager::MaybeUploadContacts() { + const ContactManager::UploadContacts* u = state_->contact_manager()->queued_upload_contacts(); + if (!u) { + return; + } + UploadContactsRequest* req = new UploadContactsRequest(this, u); + req->Start(); +} + +void NetworkManager::MaybeRemoveContacts() { + const ContactManager::RemoveContacts* r = state_->contact_manager()->queued_remove_contacts(); + if (!r) { + return; + } + RemoveContactsRequest* req = new RemoveContactsRequest(this, r); + req->Start(); +} + +void NetworkManager::MaybeUploadActivity() { + const NetworkQueue::UploadActivity* u = + state_->net_queue()->queued_upload_activity(); + if (!u) { + return; + } + if (u->activity->has_share_new() || u->activity->has_share_existing()) { + ShareRequest* req = new ShareRequest(this, u); + req->Start(); + } else if (u->activity->has_add_followers()) { + AddFollowersRequest* req = new AddFollowersRequest(this, u); + req->Start(); + } else if (u->activity->has_post_comment()) { + PostCommentRequest* req = new PostCommentRequest(this, u); + req->Start(); + } else if (u->activity->has_remove_followers()) { + RemoveFollowersRequest* req = new RemoveFollowersRequest(this, u); + req->Start(); + } else if (u->activity->has_save_photos()) { + SavePhotosRequest* req = new SavePhotosRequest(this, u); + req->Start(); + } else if (u->activity->has_unshare()) { + UnshareRequest* req = new UnshareRequest(this, u); + req->Start(); + } +} + +void NetworkManager::MaybeUploadEpisode() { + const NetworkQueue::UploadEpisode* u = + state_->net_queue()->queued_upload_episode(); + if (!u) { + return; + } + UploadEpisodeRequest* req = new UploadEpisodeRequest(this, u); + req->Start(); +} + +void NetworkManager::MaybeUploadLog() { +#if (TARGET_IPHONE_SIMULATOR) + // Don't upload logs from the simulator. + return; +#endif // (TARGET_IPHONE_SIMULATOR) + + if (!network_wifi_) { + // Only attempt log upload on wifi connections. + return; + } + + if (!state_->upload_logs()) { + // User has disabled debug log uploads + return; + } + + if (WallTime_Now() - state_->last_login_timestamp() < kUploadLogOptOutGracePeriod) { + // Uploading logs is on by default, but to ensure that users have an opportunity to + // opt-out, we don't actually start uploading logs for at least 10 minutes after login. + return; + } + + const string queue_dir = LoggingQueueDir(); + vector queued_logs; + DirList(queue_dir, &queued_logs); + if (queued_logs.empty()) { + return; + } + + // TODO(pmattis): Is the sort necessary? + std::sort(queued_logs.begin(), queued_logs.end(), std::greater()); + UploadLogRequest* req = new UploadLogRequest(this, queue_dir, queued_logs[0]); + req->Start(); +} + +void NetworkManager::MaybeUploadPhoto() { + const NetworkQueue::UploadPhoto* u = + state_->net_queue()->queued_upload_photo(); + if (!u) { + return; + } + if (u->md5.empty()) { + // The upload photo failed to queue. Call CommitQueuedUploadPhoto() on the + // network thread. + PauseLocked(NETWORK_QUEUE_SYNC); + state_->async()->dispatch_network(true, [this, u] { + LOG("network: upload photo error: %s: queueing failed", u->photo->id()); + state_->net_queue()->CommitQueuedUploadPhoto(true); + MutexLock l(&mu_); // We no longer have the lock here. + ResumeLocked(NETWORK_QUEUE_SYNC); + }); + } else { + UploadPhotoRequest* req = new UploadPhotoRequest(this, u); + req->Start(); + } +} + +void NetworkManager::StartRequestLocked(NetworkManagerQueueType queue) { + PauseLocked(queue); +} + +void NetworkManager::FinishRequest(bool success, int epoch, NetworkManagerQueueType queue) { + MutexLock l(&mu_); + if (epoch != epoch_) { + // Network requests from a different epoch should be ignored. + --queue_state_[queue].network_count; + LOG("network: ignoring network request from unexpected epoch: %d != %d", + epoch, epoch_); + return; + } + + CHECK_GT(queue_state_[queue].network_count, 0); + last_request_success_ = success; + if (success) { + queue_state_[queue].backoff_delay = kMinBackoffDelay; + ResumeLocked(queue); + } else { + BackoffLocked(queue); + } +} + +void NetworkManager::PauseLocked(NetworkManagerQueueType queue) { + mu_.AssertHeld(); + CHECK(state_->async()->Enter()); + ++queue_state_[queue].network_count; + //LOG("paused; network count: %d", network_count_); +} + +void NetworkManager::ResumeLocked(NetworkManagerQueueType queue) { + mu_.AssertHeld(); + --queue_state_[queue].network_count; + //LOG("resumed; network count: %d", network_count_); + if (!state_->async()->Exit()) { + return; + } + // Note that we intentionally do not call dispatch_main() here as we want the + // stack to unwind and locks to be released before Dispatch() is called. + state_->async()->dispatch_main_async([this] { + Dispatch(); + }); +} + +void NetworkManager::BackoffLocked(NetworkManagerQueueType queue) { + mu_.AssertHeld(); + // During the backoff delay, decrement the async running-operation + // count, so that a backed-off request does not delay destruction of + // the AsyncState object. This is especially important during + // tests, when a background thread may be generating errors due to + // unimplemented server methods and thus be in a perpetual backoff + // state. + QueueState* state = &queue_state_[queue]; + --state->network_count; + ++state->backoff_count; + NETLOG("network: backoff start: %d %.0f", state->backoff_count, state->backoff_delay); + if (!state_->async()->Exit()) { + return; + } + state_->async()->dispatch_after_main(state->backoff_delay, [this, queue, state] { + // Check the backoff_count_ flag in case the backoff was reset while we were waiting. + MutexLock lock(&mu_); + if (state->backoff_count) { + NETLOG("network: backoff done: %d", state->backoff_count); + state->backoff_delay = std::min(state->backoff_delay * 2, kMaxBackoffDelay); + ResumeFromBackoffLocked(queue); + } + }); +} + +void NetworkManager::ResumeFromBackoffLocked(NetworkManagerQueueType queue) { + mu_.AssertHeld(); + QueueState* state = &queue_state_[queue]; + CHECK_GT(state->backoff_count, 0); + CHECK(state_->async()->Enter()); + ++state->network_count; + --state->backoff_count; + ResumeLocked(queue); +} + +void NetworkManager::NuclearInvalidation(const DBHandle& updates) { + MutexLock l(&mu_); + + updates->Delete(kQueryFollowedDoneKey); + updates->Delete(kQueryFollowedLastKey); + query_followed_last_key_.clear(); + need_query_followed_ = true; +} + +bool NetworkManager::queue_is_busy(NetworkManagerQueueType queue) const { + const QueueState& state = queue_state_[queue]; + if (state.network_count || state.backoff_count) { + NETLOG("network dispatch: queue: %s count: %d, backoff: %d", + queue, state.network_count, state.backoff_count); + return true; + } + return false; +} + +void NetworkManager::ResetPing() { + last_ping_timestamp_ = 0; + dispatch_after_main(0, [this] { + Dispatch(); + }); +} + +bool NetworkManager::need_auth() const { + return state_->auth().user_cookie().empty(); +} + +const string& NetworkManager::xsrf_cookie() const { + return state_->auth().xsrf_cookie(); +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/NetworkManager.h b/clients/shared/NetworkManager.h new file mode 100644 index 0000000..7a5f77a --- /dev/null +++ b/clients/shared/NetworkManager.h @@ -0,0 +1,313 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_NETWORK_MANAGER_H +#define VIEWFINDER_NETWORK_MANAGER_H + +#import +#import "Callback.h" +#import "DB.h" +#import "Mutex.h" +#import "ScopedPtr.h" +#import "ServerUtils.h" +#import "Timer.h" +#import "WallTime.h" + +class AppState; +class NetworkManager; + +// The network manager manages several logical queues. Only one request per queue type +// can be in flight at a time. +enum NetworkManagerQueueType { + // The ping request gets its own queue because we want to block all other queues until it has completed. + NETWORK_QUEUE_PING, + + // The refresh queue performs high-priority low-bandwidth operations; mainly downloading metadata from the + // server. These operations are paused while scrolling and during similar interactions to prevent UI + // hiccups, but they may be performed while the sync queue is in the middle of a long upload or download. + // The refresh_start and refresh_end callbacks are related to operations in this queue. + NETWORK_QUEUE_REFRESH, + + // Long-polling query_notifications requests happen in their own queue. Note that an explicit refresh + // does a non-long-poll query_notifications in the REFRESH queue (which must come before this one). + NETWORK_QUEUE_NOTIFICATION, + + // The sync queue handles uploading our state to the server as well as downloads of photo data. + // The NetworkQueue priority scheme and the network_ready callback relate to operations in this queue. + // TODO(ben): Refactor PhotoManager's MaybeQueueNetwork so we can split PRIORITY_UI_* ops into a + // separate queue. + NETWORK_QUEUE_SYNC, + + NUM_NETWORK_QUEUE_TYPES, +}; + +class NetworkRequest { + friend class NetworkRequestImpl; + + public: + NetworkRequest(NetworkManager* net, NetworkManagerQueueType queue); + virtual ~NetworkRequest(); + + AppState* state() const { return state_; } + const string& url() const { return url_; } + + protected: + virtual void HandleRedirect( + ScopedPtr* new_body, + StringSet* delete_headers, StringMap* add_headers); + virtual void HandleData(const Slice& d); + virtual void HandleError(const string& e) = 0; + virtual bool HandleDone(int status_code) = 0; + + void SendGet(const string& url); + void SendPost(const string& url, const Slice& body, const Slice& content_type); + void SendPut(const string& url, const Slice& body, + const Slice& content_type, const Slice& content_md5, + const Slice& if_none_match); + void Send(const string& url, const Slice& method, const Slice& body, + const Slice& content_type, const Slice& content_md5, + const Slice& if_none_match); + + protected: + NetworkManager* const net_; + AppState* const state_; + const NetworkManagerQueueType queue_type_; + const WallTimer timer_; + string url_; + string data_; +}; + +class NetworkManager { + friend class AddFollowersRequest; + friend class AuthRequest; + friend class DownloadPhotoRequest; + friend class MergeAccountsRequest; + friend class NetworkRequest; + friend class NetworkRequestImpl; + friend class PingRequest; + friend class PostCommentRequest; + friend class QueryContactsRequest; + friend class QueryEpisodesRequest; + friend class QueryFollowedRequest; + friend class QueryNotificationsRequest; + friend class QueryUsersRequest; + friend class QueryViewpointsRequest; + friend class RecordSubscriptionRequest; + friend class RemoveFollowersRequest; + friend class RemovePhotosRequest; + friend class ResolveContactsRequest; + friend class SavePhotosRequest; + friend class ShareRequest; + friend class UnshareRequest; + friend class UpdateDeviceRequest; + friend class UpdateFriendRequest; + friend class UpdateUserRequest; + friend class UpdateUserPhotoRequest; + friend class UpdateViewpointRequest; + friend class UploadEpisodeRequest; + friend class UploadLogRequest; + friend class UploadPhotoRequest; + friend class VerifyViewfinderRequest; + + static const WallTime kMinBackoffDelay; + + public: + struct QueueState { + QueueState() + : network_count(0), + backoff_count(0), + backoff_delay(kMinBackoffDelay) { + } + int network_count; + int backoff_count; + WallTime backoff_delay; + }; + + typedef Callback AuthCallback; + typedef Callback FetchContactsCallback; + + public: + NetworkManager(AppState* state); + virtual ~NetworkManager(); + + void Dispatch(); + bool Refresh(); + + // Clear any saved error status and allow network operations to resume. + void ResetBackoff(); + + // Reset the backoff used for background query notifications. + virtual void ResetQueryNotificationsBackoff() = 0; + + // Should the application badge be cleared. + virtual bool ShouldClearApplicationBadge() = 0; + // Clear the application badge. + virtual void ClearApplicationBadge() = 0; + + // Non-queued network operations. The following methods can be + // called to start a network operation without waiting for the + // queue. + + // Attempts to resolve the given identity (which must begin with + // "Email:") and add it to the ContactManager. To see the results, + // add a contact_resolved callback on the ContactManager. + void ResolveContact(const string& identity); + + // Fetch the contacts for the specified auth service. Returns true if a fetch + // contacts request was started. + bool FetchFacebookContacts(const string& access_token, const FetchContactsCallback& done); + bool FetchGoogleContacts(const string& refresh_token, const FetchContactsCallback& done); + + // Start an authorization for a Viewfinder authorized identity (e.g. Email: + // or Phone:). In cases where the endpoint is "link" or "register", the name + // field may be non-empty. The "done" callback is invoked upon completion + // with the first parameter indicating the status code and the third a + // user-facing message from the server detailing the result. + void AuthViewfinder( + const string& endpoint, const string& identity, const string& password, + const string& first, const string& last, const string& name, + bool error_if_linked, const AuthCallback& done); + + // Verifies the identity using the specified access token. Upon receiving a + // verification link, the client clicks a link which redirects to the app or + // manually enters an access token. The "done" callback is invoked upon + // completion with the first parameter indicating the status code and the + // third a user-facing message from the server detailing the result. + void VerifyViewfinder( + const string& identity, const string& access_token, + bool manual_entry, const AuthCallback& done); + + // Submit a change password request. The "done" callback is invoked upon + // completion with the first parameter indicating the status code and the + // third a user-facing message from the server detailing the result. + void ChangePassword( + const string& old_password, const string& new_password, + const AuthCallback& done); + + // Merge with the user account specified by source_identity (retrieved by + // performing a /merge_token/viewfinder request). On completion of the merge + // operation, completion_db_key will be deleted from the database. The "done" + // callback is invoked upon completion with the first parameter indicating + // the status code and the third a user-facing message from the server + // detailing the result. + void MergeAccounts( + const string& identity, const string& access_token, + const string& completion_db_key, const AuthCallback& done); + + void SetPushNotificationDeviceToken(const string& base64_token); + // Pause/resume background operations such as querying + // notifications/viewpoints/episodes/photos. + void PauseNonInteractive(); + void ResumeNonInteractive(); + void SetNetworkDisallowed(bool disallowed); + void RunDownloadBenchmark(); + + virtual void Logout(bool clear_user_id = true); + virtual void UnlinkDevice(); + + // Prepares the NetworkManager for shutdown. Once Drain() returns no more + // requests will be started. + void Drain(); + + CallbackSet* network_changed() { return &network_changed_; } + CallbackSet* refresh_start() { return &refresh_start_; } + CallbackSet* refresh_end() { return &refresh_end_; } + bool network_up() const { return network_reachable_ || last_request_success_; } + bool network_wifi() const { return network_wifi_; } + + AppState* state() const { return state_; } + + void set_fake_401(bool val) { fake_401_ = val; } + bool fake_401() { return fake_401_; } + + bool need_auth() const; + + protected: + virtual void SetIdleTimer() = 0; + void AssetScanEnd(); + + private: + // Sub-dispatch functions for the four network queues. + void DispatchPingLocked(); + void DispatchNotificationLocked(); + void DispatchRefreshLocked(); + void DispatchSyncLocked(); + + void MaybePingServer(); + void MaybeBenchmarkDownload(); + void MaybeDownloadPhoto(); + void MaybeQueryContacts(); + void MaybeQueryEpisodes(); + void MaybeQueryFollowed(); + void MaybeQueryNotifications(bool long_poll); + void MaybeQueryUsers(); + void MaybeQueryViewpoints(); + void MaybeRecordSubscription(); + void MaybeRemoveContacts(); + void MaybeRemovePhotos(); + void MaybeUpdateDevice(); + void MaybeUpdateFriend(); + void MaybeUpdatePhoto(); + void MaybeUpdateUser(); + void MaybeUpdateViewpoint(); + void MaybeUploadActivity(); + void MaybeUploadContacts(); + void MaybeUploadEpisode(); + void MaybeUploadLog(); + void MaybeUploadPhoto(); + void StartRequestLocked(NetworkManagerQueueType queue); + void FinishRequest(bool success, int epoch, NetworkManagerQueueType queue); + void PauseLocked(NetworkManagerQueueType queue); + void ResumeLocked(NetworkManagerQueueType queue); + void BackoffLocked(NetworkManagerQueueType queue); + void ResumeFromBackoffLocked(NetworkManagerQueueType queue); + + virtual void AuthDone() = 0; + virtual void SendRequest( + NetworkRequest* req, const Slice& method, const Slice& body, + const Slice& content_type, const Slice& content_md5, + const Slice& if_none_match) = 0; + + // Clears keys for querying all followed viewpoints to rebuild all asset + // hierarchies from scratch. + void NuclearInvalidation(const DBHandle& updates); + + int epoch() { + MutexLock l(&mu_); + return epoch_; + } + + virtual bool pause_non_interactive() const = 0; + bool queue_is_busy(NetworkManagerQueueType queue) const; + + // Resets the last ping timestamp and attempts to send a new ping. + void ResetPing(); + + const string& xsrf_cookie() const; + + protected: + AppState* state_; + mutable Mutex mu_; + CallbackSet network_changed_; + CallbackSet refresh_start_; + CallbackSet refresh_end_; + QueueState queue_state_[NUM_NETWORK_QUEUE_TYPES]; + int pause_non_interactive_count_; + int epoch_; + bool last_request_success_; + bool refreshing_; + bool network_reachable_; + bool network_wifi_; + bool need_query_followed_; + string query_followed_last_key_; + bool update_device_; + bool assets_scanned_; + bool draining_; + bool network_disallowed_; + bool register_new_user_; + WallTime last_ping_timestamp_; + bool fake_401_; + std::deque benchmark_urls_; +}; + +#endif // VIEWFINDER_NETWORK_MANAGER_H diff --git a/clients/shared/NetworkQueue.cc b/clients/shared/NetworkQueue.cc new file mode 100644 index 0000000..a657f4f --- /dev/null +++ b/clients/shared/NetworkQueue.cc @@ -0,0 +1,2357 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "AppState.h" +#import "AsyncState.h" +#import "ContactManager.h" +#import "FileUtils.h" +#import "NetworkManager.h" +#import "NetworkQueue.h" +#import "NotificationManager.h" +#import "PathUtils.h" +#import "PhotoStorage.h" +#import "PlacemarkHistogram.h" +#import "ScopedPtr.h" +#import "Server.pb.h" +#import "ServerUtils.h" +#import "StringUtils.h" +#import "Timer.h" + +namespace { + +const string kNextSequenceKey = DBFormat::metadata_key("next_network_queue_sequence"); +const string kNetworkQueueKeyPrefix = DBFormat::network_queue_key(""); + +const int kMaxPhotosPerUpload = 10; + +const DBRegisterKeyIntrospect kNetworkQueueKeyIntrospect( + kNetworkQueueKeyPrefix, [](Slice key) { + int priority; + int64_t sequence; + if (!DecodeNetworkQueueKey(key, &priority, &sequence)) { + return string(); + } + return string(Format("%d/%d", priority, sequence)); + }, NULL); + +// Returns the adjustment that should be made to the activity count for an operation +// at the given priority (which we use for a crude approximation of the type of operation). +double AdjustedCountForPriority(int priority) { + switch (priority) { + // Uploading one photo results in four operations, so count each one as 0.25. + // If "store originals" is turned on, there is a fifth operation (which counts as 1.0 + // so the count in this case will be two per photo). This is reasonable since we + // present "store originals" as a separate feature, and otherwise we'd have to do something + // special when backfilling originals. + case PRIORITY_UI_UPLOAD_PHOTO: + case PRIORITY_UPLOAD_PHOTO: + case PRIORITY_UPLOAD_PHOTO_MEDIUM: + return 0.25; + default: + return 1; + } +} + +} // namespace + +string EncodeNetworkQueueKey(int priority, int64_t sequence) { + string s(kNetworkQueueKeyPrefix); + OrderedCodeEncodeVarint32(&s, priority); + OrderedCodeEncodeVarint64(&s, sequence); + return s; +} + +bool DecodeNetworkQueueKey(Slice key, int* priority, int64_t* sequence) { + if (!key.starts_with(kNetworkQueueKeyPrefix)) { + return false; + } + key.remove_prefix(kNetworkQueueKeyPrefix.size()); + *priority = OrderedCodeDecodeVarint32(&key); + *sequence = OrderedCodeDecodeVarint64(&key); + return true; +} + +NetworkQueue::Iterator::Iterator(leveldb::Iterator* iter) + : iter_(iter), + done_(false), + priority_(0), + sequence_(0) { + iter_->Seek(kNetworkQueueKeyPrefix); + while (!done_ && !UpdateState()) { + iter_->Next(); + } +} + +NetworkQueue::Iterator::~Iterator() { +} + +void NetworkQueue::Iterator::Next() { + while (!done_) { + iter_->Next(); + if (UpdateState()) { + break; + } + } +} + +void NetworkQueue::Iterator::SkipPriority() { + if (done_) { + return; + } + const string next_priority_key = EncodeNetworkQueueKey(priority() + 1, 0); + iter_->Seek(next_priority_key); + while (!done_ && !UpdateState()) { + iter_->Next(); + } +} + +bool NetworkQueue::Iterator::UpdateState() { + op_.Clear(); + if (!iter_->Valid()) { + done_ = true; + return true; + } + const Slice key(ToSlice(iter_->key())); + if (!DecodeNetworkQueueKey(key, &priority_, &sequence_)) { + done_ = true; + } + const Slice value(ToSlice(iter_->value())); + if (!op_.ParseFromArray(value.data(), value.size())) { + return false; + } + return true; +} + +NetworkQueue::NetworkQueue(AppState* state) + : state_(state), + next_sequence_(0), + queue_in_progress_(false), + queue_start_time_(0), + photo_tmp_dir_(JoinPath(TmpDir(), "photos")) { + // Remove the photo tmp directory and all of its contents and recreate it. + DirRemove(photo_tmp_dir_, true); + DirCreate(photo_tmp_dir_); + + state_->network_ready()->Add([this](int priority) { + MaybeQueueNetwork(priority); + }); + + // Set up callbacks for handling notification mgr callbacks. + state_->notification_manager()->process_notifications()->Add( + [this](const QueryNotificationsResponse& p, const DBHandle& updates) { + ProcessQueryNotifications(p, updates); + }); + state_->notification_manager()->nuclear_invalidations()->Add( + [this](const DBHandle& updates) { + // A nuclear invalidation. Clear all of the existing invalidations. + state_->viewpoint_table()->ClearAllInvalidations(updates); + state_->episode_table()->ClearAllInvalidations(updates); + }); + + // The DB might not have been opened at this point, so don't access it yet. +} + +NetworkQueue::~NetworkQueue() { +} + +int64_t NetworkQueue::Add( + int priority, const ServerOperation& op, const DBHandle& updates) { + MutexLock l(&mu_); + EnsureInitLocked(); + const int64_t sequence = next_sequence_++; + state_->db()->Put(kNextSequenceKey, next_sequence_); + updates->PutProto(EncodeNetworkQueueKey(priority, sequence), op); + UpdateStatsLocked(priority, op, true); + + updates->AddCommitTrigger("network", [this] { + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + }); + return sequence; +} + +void NetworkQueue::Remove( + int priority, int64_t sequence, const DBHandle& updates) { + const string key = EncodeNetworkQueueKey(priority, sequence); + ServerOperation op; + if (!updates->GetProto(key, &op)) { + return; + } + MutexLock l(&mu_); + updates->Delete(key); + UpdateStatsLocked(priority, op, false); + + updates->AddCommitTrigger("network", [this] { + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + }); +} + +void NetworkQueue::Remove( + int priority, int64_t sequence, + const ServerOperation& op, const DBHandle& updates) { + MutexLock l(&mu_); + updates->Delete(EncodeNetworkQueueKey(priority, sequence)); + UpdateStatsLocked(priority, op, false); +} + +bool NetworkQueue::QueuePhoto(const PhotoHandle& ph, const DBHandle& updates) { + if (ph->label_error() || + ph->candidate_duplicates_size() > 0) { + // The photo is quarantined or otherwise non-uploadable. Remove it from the + // queue. + return DequeuePhoto(ph, updates); + } + + // Set the priority if it hasn't been set or is less than the currently set + // priority. Otherwise, add the priority as a stat. +#define MAYBE_SET_PRIORITY(r, p) \ + if (!priority || (p) < priority) { \ + if (priority > 0) { \ + op.add_stats(priority); \ + } \ + priority = p; \ + reason = r; \ + } else { \ + op.add_stats(p); \ + } + + ServerOperation op; + int priority = 0; + const char* reason = NULL; + if (ph->download_thumbnail()) { + MAYBE_SET_PRIORITY( + "download_thumbnail", + ph->error_ui_thumbnail() ? PRIORITY_UI_THUMBNAIL : PRIORITY_DOWNLOAD_PHOTO); + } + if (ph->download_full()) { + MAYBE_SET_PRIORITY( + "download_full", + ph->error_ui_full() ? PRIORITY_UI_FULL : PRIORITY_DOWNLOAD_PHOTO); + } + if (ph->download_original()) { + MAYBE_SET_PRIORITY( + "download_original", + ph->error_ui_original() ? PRIORITY_UI_ORIGINAL : PRIORITY_DOWNLOAD_PHOTO); + } + if (ph->upload_metadata()) { + MAYBE_SET_PRIORITY( + "upload_metadata", + ph->shared() ? PRIORITY_UI_UPLOAD_PHOTO : PRIORITY_UPLOAD_PHOTO); + } else if (ph->update_metadata()) { + MAYBE_SET_PRIORITY( + "update_metadata", + ph->shared() ? PRIORITY_UI_UPLOAD_PHOTO : PRIORITY_UPLOAD_PHOTO); + } + if (ph->upload_thumbnail()) { + MAYBE_SET_PRIORITY( + "upload_thumbnail", + ph->shared() ? PRIORITY_UI_UPLOAD_PHOTO : PRIORITY_UPLOAD_PHOTO); + } + if (ph->upload_full()) { + MAYBE_SET_PRIORITY( + "upload_full", + ph->shared() ? PRIORITY_UI_UPLOAD_PHOTO : PRIORITY_UPLOAD_PHOTO); + } + if (ph->upload_medium()) { + MAYBE_SET_PRIORITY( + "upload_medium", + ph->shared() ? PRIORITY_UI_UPLOAD_PHOTO : PRIORITY_UPLOAD_PHOTO_MEDIUM); + } + if (ph->upload_original()) { + // Note: original images are only uploaded when cloud storage is enabled + // and are never given PRIORITY_UI_UPLOAD_PHOTO. + MAYBE_SET_PRIORITY("upload_original", PRIORITY_UPLOAD_PHOTO_ORIGINAL); + } + +#undef MAYBE_SET_PRIORITY + + bool changed = false; + const int64_t old_priority = ph->queue().priority(); + const int64_t old_sequence = ph->queue().sequence(); + // Always dequeue the photo, even if the priority isn't changing, in order to + // properly update the network queue stats. + changed = DequeuePhoto(ph, updates); + if (priority > 0) { + op.set_update_photo(ph->id().local_id()); + ph->mutable_queue()->set_priority(priority); + // If the priority of the photo is unchanged, reuse the old sequence + // number. This is essential to make sure that we process all of the + // updates for a photo sequentially instead of constantly moving the photo + // to the end of the queue. + if (priority == old_priority) { + MutexLock l(&mu_); + updates->PutProto(EncodeNetworkQueueKey(priority, old_sequence), op); + UpdateStatsLocked(priority, op, true); + ph->mutable_queue()->set_sequence(old_sequence); + } else { + ph->mutable_queue()->set_sequence(Add(priority, op, updates)); + } + changed = true; + VLOG("queueing %s (%s): %d,%d", ph->id(), reason, + ph->queue().priority(), ph->queue().sequence()); + } + + return changed; +} + +bool NetworkQueue::DequeuePhoto(const PhotoHandle& ph, const DBHandle& updates) { + if (!ph->has_queue()) { + return false; + } + VLOG("dequeuing %s: %d,%d", ph->id(), + ph->queue().priority(), ph->queue().sequence()); + Remove(ph->queue().priority(), ph->queue().sequence(), updates); + ph->clear_queue(); + return true; +} + +bool NetworkQueue::QueueActivity(const ActivityHandle& ah, const DBHandle& updates) { + if (ah->label_error() || ah->provisional()) { + // The activity is quarantined. Remove it from the queue. + return DequeueActivity(ah, updates); + } + + int priority = 0; + const char* reason = NULL; + if (ah->upload_activity()) { + priority = PRIORITY_UI_ACTIVITY; + reason = "upload_activity"; + } + + // Always pedantically dequeue and queue the activity upload. This is less + // fragile than relying on other code to have removed any existing queue + // metadata after the last op was run successfully. + const int64_t old_priority = ah->queue().priority(); + const int64_t old_sequence = ah->queue().sequence(); + bool changed = DequeueActivity(ah, updates); + if (priority > 0) { + ServerOperation op; + op.mutable_headers()->set_op_id(state_->NewLocalOperationId()); + op.mutable_headers()->set_op_timestamp(WallTime_Now()); + op.set_upload_activity(ah->activity_id().local_id()); + ah->mutable_queue()->set_priority(priority); + // If the priority of the activity is unchanged, reuse the old sequence + // number. This is essential to make sure that we process activities in the + // order they are generated instead of constantly moving them to the end of + // the queue. + if (priority == old_priority) { + // TODO(peter): Share this code with QueuePhoto() and QueueViewpoint(). + MutexLock l(&mu_); + updates->PutProto(EncodeNetworkQueueKey(priority, old_sequence), op); + UpdateStatsLocked(priority, op, true); + ah->mutable_queue()->set_sequence(old_sequence); + } else { + ah->mutable_queue()->set_sequence(Add(priority, op, updates)); + } + changed = true; + VLOG("queueing %s (%s): %d,%d", ah->activity_id(), reason, + ah->queue().priority(), ah->queue().sequence()); + } + + return changed; +} + +bool NetworkQueue::DequeueActivity(const ActivityHandle& ah, const DBHandle& updates) { + if (!ah->has_queue()) { + return false; + } + VLOG("dequeuing %s: %d,%d", ah->activity_id(), + ah->queue().priority(), ah->queue().sequence()); + Remove(ah->queue().priority(), ah->queue().sequence(), updates); + ah->clear_queue(); + return true; +} + +bool NetworkQueue::QueueViewpoint(const ViewpointHandle& vh, const DBHandle& updates) { + if (vh->label_error() || vh->provisional()) { + // The viewpoint is quarantined. Remove it from the queue. + return DequeueViewpoint(vh, updates); + } + + int priority = 0; + const char* reason = NULL; + if (vh->update_metadata() || + vh->update_follower_metadata() || + vh->update_remove() || + vh->update_viewed_seq()) { + priority = PRIORITY_UPDATE_VIEWPOINT; + reason = "update_viewpoint"; + } + + // Always pedantically dequeue and queue the viewpoint update. This + // is less fragile than relying on other code to have removed any + // existing queue metadata after the last op was run successfully. + const int64_t old_priority = vh->queue().priority(); + const int64_t old_sequence = vh->queue().sequence(); + bool changed = DequeueViewpoint(vh, updates); + if (priority > 0) { + ServerOperation op; + op.mutable_headers()->set_op_id(state_->NewLocalOperationId()); + op.mutable_headers()->set_op_timestamp(WallTime_Now()); + op.set_update_viewpoint(vh->id().local_id()); + vh->mutable_queue()->set_priority(priority); + // If the priority of the viewpoint is unchanged, reuse the old sequence + // number. This is essential to make sure that we process viewpoint updates + // in the order they are generated instead of constantly moving them to the + // end of the queue. + if (priority == old_priority) { + // TODO(peter): Share this code with QueuePhoto() and QueueActivity(). + MutexLock l(&mu_); + updates->PutProto(EncodeNetworkQueueKey(priority, old_sequence), op); + UpdateStatsLocked(priority, op, true); + vh->mutable_queue()->set_sequence(old_sequence); + } else { + vh->mutable_queue()->set_sequence(Add(priority, op, updates)); + } + changed = true; + VLOG("queueing viewpoint %s (%s): %d,%d", vh->id(), reason, + vh->queue().priority(), vh->queue().sequence()); + } + + return changed; +} + +bool NetworkQueue::DequeueViewpoint(const ViewpointHandle& vh, const DBHandle& updates) { + if (!vh->has_queue()) { + return false; + } + VLOG("dequeuing viewpoint %s: %d,%d", vh->id(), + vh->queue().priority(), vh->queue().sequence()); + Remove(vh->queue().priority(), vh->queue().sequence(), updates); + vh->clear_queue(); + return true; +} + +NetworkQueue::Iterator* NetworkQueue::NewIterator() { + return new Iterator(state_->db()->NewIterator()); +} + +bool NetworkQueue::Empty() { + return TopPriority() == -1; +} + +int NetworkQueue::TopPriority() { + for (ScopedPtr iter(NewIterator()); + !iter->done(); + iter->SkipPriority()) { + if (ShouldProcessPriority(iter->priority())) { + return iter->priority(); + } + } + return -1; +} + +int NetworkQueue::GetNetworkCount() { + MutexLock l(&mu_); + EnsureStatsInitLocked(); + + double count = 0; + for (NetworkStatsMap::const_iterator iter(stats_->begin()); + iter != stats_->end(); + ++iter) { + const int priority = iter->first; + if (ShouldProcessPriority(priority)) { + count += iter->second; + } + } + return ceil(count); +} + +int NetworkQueue::GetDownloadCount() { + MutexLock l(&mu_); + EnsureStatsInitLocked(); + + double count = 0; + for (NetworkStatsMap::const_iterator iter(stats_->begin()); + iter != stats_->end(); + ++iter) { + const int priority = iter->first; + if (ShouldProcessPriority(priority) && + IsDownloadPriority(priority)) { + count += iter->second; + } + } + return ceil(count); +} + +int NetworkQueue::GetUploadCount() { + MutexLock l(&mu_); + EnsureStatsInitLocked(); + + double count = 0; + for (NetworkStatsMap::const_iterator iter(stats_->begin()); + iter != stats_->end(); + ++iter) { + const int priority = iter->first; + if (ShouldProcessPriority(priority) && + !IsDownloadPriority(priority)) { + count += iter->second; + } + } + return ceil(count); +} + +bool NetworkQueue::ShouldProcessPriority(int priority) const { + if (priority == PRIORITY_UPLOAD_PHOTO || + priority == PRIORITY_UPLOAD_PHOTO_MEDIUM || + priority == PRIORITY_UPLOAD_PHOTO_ORIGINAL) { + if (!state_->CloudStorageEnabled()) { + return false; + } + // TODO(peter): Add a setting to control whether we upload photos over + // 3g/lte when cloud storage is enabled. + if (!state_->network_wifi()) { + return false; + } + } + if (priority == PRIORITY_UPLOAD_PHOTO_ORIGINAL && + !state_->store_originals()) { + return false; + } + if (priority == PRIORITY_DOWNLOAD_PHOTO && + !state_->network_wifi()) { + return false; + } + return true; +} + +bool NetworkQueue::IsDownloadPriority(int priority) const { + return priority == PRIORITY_UI_THUMBNAIL || + priority == PRIORITY_UI_FULL || + priority == PRIORITY_UI_ORIGINAL || + priority == PRIORITY_DOWNLOAD_PHOTO; +} + +void NetworkQueue::CommitQueuedDownloadPhoto(const string& md5, bool retry) { + if (!queued_download_photo_.get()) { + LOG("photo: commit failed: no photo download queued"); + return; + } + DownloadPhoto* const d = queued_download_photo_.get(); + + string filename; + switch (d->type) { + case THUMBNAIL: + filename = PhotoThumbnailFilename(d->photo->id()); + break; + case MEDIUM: + filename = PhotoMediumFilename(d->photo->id()); + break; + case FULL: + filename = PhotoFullFilename(d->photo->id()); + break; + case ORIGINAL: + filename = PhotoOriginalFilename(d->photo->id()); + break; + } + + const bool error = md5.empty() && !retry; + if (!error) { + DBHandle updates = state_->NewDBTransaction(); + + if (state_->photo_storage()->AddExisting( + d->path, filename, md5, d->photo->id().server_id(), updates)) { + // Clear the download bits on success. + d->photo->Lock(); + + switch (d->type) { + case THUMBNAIL: + d->photo->clear_download_thumbnail(); + d->photo->clear_error_download_thumbnail(); + d->photo->clear_error_ui_thumbnail(); + break; + case MEDIUM: + d->photo->clear_download_medium(); + d->photo->clear_error_download_medium(); + break; + case FULL: + d->photo->clear_download_full(); + d->photo->clear_error_download_full(); + d->photo->clear_error_ui_full(); + break; + case ORIGINAL: + d->photo->clear_download_original(); + d->photo->clear_error_download_original(); + d->photo->clear_error_ui_original(); + break; + } + + d->photo->SaveAndUnlock(updates); + updates->Commit(); + } else { + retry = true; + } + } + + if (!retry) { + // Run any download callbacks (on both success and error) after the + // downloaded photo has been written. + NotifyDownload(d->photo->id().local_id(), d->type); + } + + if (error) { + // A persistent error (e.g. photo does not exist). Stop trying to download + // the photo. + DownloadPhotoError(d->photo, d->type); + } + + queued_download_photo_.reset(NULL); +} + +void NetworkQueue::CommitQueuedRemovePhotos(bool error) { + if (!queued_remove_photos_.get()) { + LOG("photo: commit failed: no remove photos queued"); + return; + } + + // TODO(pmattis): How to handle errors? We tried to remove the photos but the + // server returned an unrecoverable error. + + RemovePhotos* r = queued_remove_photos_.get(); + + DBHandle updates = state_->NewDBTransaction(); + // Note, passing in the empty ServerOperation() is okay because the + // RemovePhotos operation has no sub operations. + Remove(r->queue.priority(), r->queue.sequence(), ServerOperation(), updates); + updates->Commit(); + + queued_remove_photos_.reset(NULL); +} + +void NetworkQueue::CommitQueuedUpdatePhoto(bool error) { + if (!queued_update_photo_.get()) { + LOG("photo: commit failed: no photo update queued"); + return; + } + + PhotoHandle ph = queued_update_photo_->photo; + queued_update_photo_.reset(NULL); + + if (error) { + UpdatePhotoError(ph); + return; + } + + DBHandle updates = state_->NewDBTransaction(); + ph->Lock(); + ph->clear_update_metadata(); + ph->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::CommitQueuedUpdateViewpoint(UpdateViewpointType type, bool error) { + if (!queued_update_viewpoint_.get()) { + LOG("photo: commit failed: no viewpoint update queued"); + return; + } + + ViewpointHandle vh = queued_update_viewpoint_->viewpoint; + queued_update_viewpoint_.reset(NULL); + + if (error) { + UpdateViewpointError(vh); + return; + } + + DBHandle updates = state_->NewDBTransaction(); + vh->Lock(); + switch (type) { + case UPDATE_VIEWPOINT_METADATA: + vh->clear_update_metadata(); + break; + case UPDATE_VIEWPOINT_FOLLOWER_METADATA: + vh->clear_update_follower_metadata(); + break; + case UPDATE_VIEWPOINT_REMOVE: + vh->clear_update_remove(); + break; + case UPDATE_VIEWPOINT_VIEWED_SEQ: + vh->clear_update_viewed_seq(); + break; + default: + DCHECK(false) << "unknown update viewpoint type " << type; + LOG("photo: unknown update viewpoint type: %d", type); + } + vh->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::CommitQueuedUploadEpisode( + const UploadEpisodeResponse& r, int status) { + if (!queued_upload_episode_.get()) { + LOG("photo: commit failed: no episode upload queued"); + return; + } + + UploadEpisode* u = queued_upload_episode_.get(); + if (status != 200) { + // Episode upload failed. + DBHandle updates = state_->NewDBTransaction(); + EpisodeHandle eh = u->episode; + for (int i = 0; i < u->photos.size(); ++i) { + const PhotoHandle& ph = u->photos[i]; + ph->Lock(); + if (ph->error_upload_metadata()) { + // We had previously failed trying to upload this photo's + // metadata. Quarantine the photo so that we don't try again. + ph->Unlock(); + QuarantinePhoto(ph, "upload: metadata", updates); + continue; + } + ph->set_error_upload_metadata(true); + ph->SaveAndUnlock(updates); + } + queued_upload_episode_.reset(NULL); + // Query the server for the episode metadata again. This will refresh the + // metadata for all the photos in the episode, resetting their state. If we + // then attempt to upload their metadata again we'll encounter the + // error_upload_metadata bit and quarantine the photo. + EpisodeSelection s; + s.set_episode_id(eh->id().server_id()); + s.set_get_attributes(true); + s.set_get_photos(true); + state_->episode_table()->Invalidate(s, updates); + updates->Commit(); + return; + } + + if (u->photos.size() != r.photos_size()) { + LOG("photo: commit failed: unexpected response size"); + queued_upload_episode_.reset(NULL); + return; + } + + DBHandle updates = state_->NewDBTransaction(); + + for (int i = 0; i < r.photos_size(); ++i) { + const PhotoHandle& ph = u->photos[i]; + const PhotoUpdate& u = r.photos(i); + if (ph->id().server_id() != u.metadata().id().server_id()) { + LOG("photo: unexpected server id in response: %s != %s", + ph->id(), u.metadata().id()); + continue; + } + ProcessPhoto(ph, r.photos(i), NULL, updates); + ph->SaveAndUnlock(updates); + } + + updates->Commit(); + + queued_upload_episode_.reset(NULL); +} + +void NetworkQueue::CommitQueuedUploadPhoto(bool error) { + if (!queued_upload_photo_.get()) { + LOG("photo: commit failed: no photo upload queued"); + return; + } + + PhotoHandle ph = queued_upload_photo_->photo; + const PhotoType type = queued_upload_photo_->type; + const string path = queued_upload_photo_->path; + queued_upload_photo_.reset(NULL); + + if (error) { + if (ph->GetDeviceId() == state_->device_id()) { + // Only indicate upload errors on photos that were created on the current + // device. If the photo was not created on the current device, we assume + // that the server already has the photo and that the upload error was + // because of a spurious content-md5 mismatch. + UploadPhotoError(ph, type); + return; + } + VLOG("photo: %s unable to upload photo created by device %d (current device %d)", + ph->id(), ph->GetDeviceId(), state_->device_id()); + } + + DBHandle updates = state_->NewDBTransaction(); + bool delete_photo = false; + ph->Lock(); + + // Clear the upload error bit on success and delete the associated put url. + switch (type) { + case THUMBNAIL: + ph->DeleteURL("tn_put", updates); + ph->clear_upload_thumbnail(); + ph->clear_error_upload_thumbnail(); + break; + case MEDIUM: + ph->DeleteURL("med_put", updates); + ph->clear_upload_medium(); + ph->clear_error_upload_medium(); + delete_photo = true; + break; + case FULL: + ph->DeleteURL("full_put", updates); + ph->clear_upload_full(); + ph->clear_error_upload_full(); + delete_photo = ph->HasAssetUrl(); + break; + case ORIGINAL: + ph->DeleteURL("orig_put", updates); + ph->clear_upload_original(); + ph->clear_error_upload_original(); + delete_photo = true; + break; + } + + const string filename = PhotoBasename(state_->photo_dir(), path); + for (int i = 0; i < ph->asset_keys_size(); i++) { + Slice fingerprint; + if (!DecodeAssetKey(ph->asset_keys(i), NULL, &fingerprint)) { + continue; + } + // The photo has been uploaded to the server. Store a symlink to the asset + // key it is associated with in the photo server directory so that we can + // avoid having to try to upload the photo again if the database format + // changes. + state_->photo_storage()->SetAssetSymlink( + filename, ph->id().server_id(), + fingerprint.ToString()); + DCHECK_EQ(fingerprint, state_->photo_storage()->ReadAssetSymlink( + filename, ph->id().server_id())); + } + + if (delete_photo && !path.empty()) { + // The photo has been uploaded to the server, no need to keep the + // original/medium images around. + state_->photo_storage()->Delete(filename, updates); + } else if (!ph->HasAssetUrl()) { + // The photo was successfully uploaded to the server. Link it to the server + // photo directory so that we can avoid downloading the photo again if the + // database format changes. + state_->photo_storage()->SetServerId( + filename, ph->id().server_id(), updates); + } + + ph->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::CommitQueuedUploadActivity(bool error) { + if (!queued_upload_activity_.get()) { + LOG("photo: commit failed: no activity upload queued"); + return; + } + + ActivityHandle ah = queued_upload_activity_->activity; + queued_upload_activity_.reset(NULL); + + if (error) { + UploadActivityError(ah); + return; + } + + DBHandle updates = state_->NewDBTransaction(); + ah->Lock(); + ah->clear_upload_activity(); + ah->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::ProcessQueryEpisodes( + const QueryEpisodesResponse& r, const vector& v, + const DBHandle& updates) { + typedef std::unordered_map EpisodeMap; + + for (int i = 0; i < v.size(); ++i) { + state_->episode_table()->Validate(v[i], updates); + } + + for (int i = 0; i < r.episodes_size(); ++i) { + const QueryEpisodesResponse::Episode& e = r.episodes(i); + EpisodeMap old_episodes; + + EpisodeHandle eh; + if (e.has_metadata()) { + eh = ProcessEpisode(e.metadata(), false, updates); + } + + for (int j = 0; j < e.photos_size(); ++j) { + EpisodeHandle old_eh; + PhotoHandle ph = ProcessPhoto(e.photos(j), &old_eh, updates); + if (eh.get()) { + const PhotoMetadata& m = e.photos(j).metadata(); + // Note: Even though we just merged "m" into "ph", we cleared + // label_removed() and label_unshared() because those labels exist only + // on the relationship between the photo and the episode we're + // processing. So it's critically important to use "m.label_removed()" + // and "m.label_unshared()" in the expression below. + if (m.label_unshared()) { + eh->UnsharePhoto(ph->id().local_id()); + } else if (m.label_hidden()) { + eh->HidePhoto(ph->id().local_id()); + } else if (m.label_removed()) { + eh->RemovePhoto(ph->id().local_id()); + } else { + eh->AddPhoto(ph->id().local_id()); + // If the photo has an error, try to unquarantine the photo as + // this episode may indicate a route through which the photo + // may be successfully loaded and permanently unquarantined. + // Dispatch on main thread to avoid re-entering episode locks. + if (ph->label_error()) { + const int64_t photo_id = ph->id().local_id(); + state_->async()->dispatch_after_low_priority(0, [this, photo_id] { + state_->photo_table()->MaybeUnquarantinePhoto(photo_id); + }); + } + } + + // If this is the canonical episode for the photo, make sure the local + // id is correct in PhotoMetadata. + if (ph->episode_id().server_id() == eh->id().server_id()) { + ph->mutable_episode_id()->CopyFrom(eh->id()); + } + } + if (old_eh.get()) { + // The photo is changing episodes. Remove it from the old episode. We + // have to perform this removal after the addition of the photo to the + // new episode to ensure that the photo is always referenced by an + // episode so that its images and assets do not get deleted. + EpisodeHandle& saved_eh = old_episodes[old_eh->id().local_id()]; + if (!saved_eh.get()) { + saved_eh = old_eh; + saved_eh->Lock(); + } + saved_eh->RemovePhoto(ph->id().local_id()); + } + ph->SaveAndUnlock(updates); + } + + if (eh.get()) { + eh->SaveAndUnlock(updates); + } + + // Only save the old episodes after the new episode has been saved. + for (EpisodeMap::iterator iter(old_episodes.begin()); + iter != old_episodes.end(); + ++iter) { + iter->second->SaveAndUnlock(updates); + } + } +} + +void NetworkQueue::ProcessQueryFollowed( + const QueryFollowedResponse& r, const DBHandle& updates) { + for (int i = 0; i < r.viewpoints_size(); ++i) { + ViewpointHandle vh = ProcessViewpoint(r.viewpoints(i), true, updates); + vh->SaveAndUnlock(updates); + } +} + +void NetworkQueue::ProcessQueryNotifications( + const QueryNotificationsResponse& r, const DBHandle& updates) { + // LOG("process query notifications: %s", r); + UsageMetadata merged_usage; + bool found_remote_usage = false; + for (int i = 0; i < r.notifications_size(); ++i) { + const QueryNotificationsResponse::Notification& n = r.notifications(i); + if (n.has_invalidate()) { + const InvalidateMetadata& invalidate = n.invalidate(); + for (int j = 0; j < invalidate.viewpoints_size(); ++j) { + state_->viewpoint_table()->Invalidate(invalidate.viewpoints(j), updates); + VLOG("notification %d invalidated viewpoint %s", + n.notification_id(), invalidate.viewpoints(j).viewpoint_id()); + } + for (int j = 0; j < invalidate.episodes_size(); ++j) { + state_->episode_table()->Invalidate(invalidate.episodes(j), updates); + VLOG("notification %d invalidated episode %s", + n.notification_id(), invalidate.episodes(j).episode_id()); + } + + // NOTE: contacts invalidation is handled in + // ContactManager.ProcessQueryNotifications. + } + if (n.has_inline_invalidate()) { + if (n.inline_invalidate().has_activity()) { + // Process the activity. + ActivityHandle ah = ProcessActivity(n.inline_invalidate().activity(), updates); + ah->SaveAndUnlock(updates); + VLOG("notification %d added activity %s", n.notification_id(), ah->activity_id()); + } + if (n.inline_invalidate().has_viewpoint()) { + // Set the update_seq and viewed_seq values on the indicated + // viewpoint. If the viewpoint doesn't exist yet, we've + // probably already written an invalidation for it, but + // haven't yet queried it. When it's queried, we'll receive + // up-to-date viewed/update sequence values. + const QueryNotificationsResponse::InlineViewpoint& iv = + n.inline_invalidate().viewpoint(); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + iv.viewpoint_id(), updates); + if (vh.get()) { + vh->Lock(); + if (iv.has_update_seq() && iv.update_seq() > vh->update_seq()) { + vh->set_update_seq(iv.update_seq()); + } + if (iv.has_viewed_seq() && iv.viewed_seq() > vh->viewed_seq()) { + vh->set_viewed_seq(iv.viewed_seq()); + } + vh->SaveAndUnlock(updates); + VLOG("notification %d added viewpoint %s", n.notification_id(), vh->id()); + } + } + if (n.inline_invalidate().has_comment()) { + // An inlined comment. Create the comment directly. + CommentHandle ch = ProcessComment(n.inline_invalidate().comment(), updates); + ch->SaveAndUnlock(updates); + VLOG("added comment %s to %s", ch->comment_id(), ch->viewpoint_id()); + } + if (n.inline_invalidate().has_usage()) { + // inlined usage. It is always optional and may be partial (eg: owned-by only). + merged_usage.MergeFrom(n.inline_invalidate().usage()); + found_remote_usage = true; + } + } + } + // Only update remote usage once we've merged all entries. This saves db Puts. + if (found_remote_usage) { + VLOG("Merged usage: %s", ToString(merged_usage)); + state_->photo_storage()->update_remote_usage(merged_usage); + } +} + +void NetworkQueue::ProcessQueryViewpoints( + const QueryViewpointsResponse& r, const vector& v, + const DBHandle& updates) { + for (int i = 0; i < v.size(); ++i) { + state_->viewpoint_table()->Validate(v[i], updates); + } + + for (int i = 0; i < r.viewpoints_size(); ++i) { + const QueryViewpointsResponse::Viewpoint& v = r.viewpoints(i); + + ViewpointHandle vh; + if (v.has_metadata()) { + vh = ProcessViewpoint(v.metadata(), false, updates); + VLOG("viewpoint %s", vh->id()); + } else { + LOG("photo: ERROR: viewpoint id not returned with /query_viewpoints"); + } + + if (vh.get() && v.followers_size() > 0) { + for (int j = 0; j < v.followers_size(); ++j) { + if (v.followers(j).has_label_removed() && + v.followers(j).has_label_unrevivable()) { + vh->RemoveFollower(v.followers(j).follower_id()); + VLOG("removed follower from %s: %d", vh->id(), v.followers(j).follower_id()); + } else { + state_->contact_manager()->MaybeQueueUser(v.followers(j).follower_id(), updates); + vh->AddFollower(v.followers(j).follower_id()); + VLOG("added follower to %s: %d", vh->id(), v.followers(j).follower_id()); + } + } + } + + for (int j = 0; j < v.activities_size(); ++j) { + ActivityHandle ah = ProcessActivity(v.activities(j), updates); + // If the activity has already been viewed, set viewed timestamp. + ah->SaveAndUnlock(updates); + // Add a follower if the activity is meant to indicate a merged user account. + if (vh.get() && ah->has_merge_accounts()) { + vh->AddFollower(ah->merge_accounts().target_user_id()); + } + VLOG("added activity %s to %s", ah->activity_id(), vh->id()); + } + + for (int j = 0; j < v.episodes_size(); ++j) { + EpisodeHandle eh = ProcessEpisode(v.episodes(j), true, updates); + eh->SaveAndUnlock(updates); + VLOG("added episode %s to %s", eh->id(), vh->id()); + } + + for (int j = 0; j < v.comments_size(); ++j) { + CommentHandle ch = ProcessComment(v.comments(j), updates); + ch->SaveAndUnlock(updates); + VLOG("added comment %s to %s", ch->comment_id(), vh->id()); + } + + if (vh.get()) { + vh->SaveAndUnlock(updates); + } + } +} + +void NetworkQueue::WaitForDownload( + int64_t photo_id, PhotoType desired_type, Callback done) { + MutexLock l(&download_callback_mu_); + DownloadCallbackSet*& callbacks = download_callback_map_[photo_id]; + if (!callbacks) { + callbacks = new DownloadCallbackSet; + } + int* callback_id = new int; + *callback_id = callbacks->Add( + [this, photo_id, desired_type, done, callback_id](int type) { + if ((type & desired_type) == 0) { + return; + } + // Mutex lock is held by caller. + download_callback_mu_.AssertHeld(); + download_callback_map_[photo_id]->Remove(*callback_id); + delete callback_id; + done(); + }); +} + +void NetworkQueue::EnsureInitLocked() { + if (!next_sequence_) { + next_sequence_ = state_->db()->Get(kNextSequenceKey, 1); + } +} + +NetworkQueue::NetworkStatsMap NetworkQueue::stats() { + MutexLock l(&mu_); + EnsureStatsInitLocked(); + return *stats_; +} + +void NetworkQueue::UpdateStatsLocked( + int priority, const ServerOperation& op, bool addition) { + EnsureStatsInitLocked(); + if (addition) { + (*stats_)[priority] += AdjustedCountForPriority(priority); + for (int i = 0; i < op.stats_size(); ++i) { + (*stats_)[op.stats(i)] += AdjustedCountForPriority(op.stats(i)); + } + } else { + (*stats_)[priority] -= AdjustedCountForPriority(priority); + if ((*stats_)[priority] <= 0) { + stats_->erase(priority); + } + for (int i = 0; i < op.stats_size(); ++i) { + const int p = op.stats(i); + (*stats_)[p] -= AdjustedCountForPriority(p); + if ((*stats_)[p] <= 0) { + stats_->erase(p); + } + } + } +} + +void NetworkQueue::EnsureStatsInitLocked() { + if (stats_.get()) { + return; + } + + WallTimer timer; + stats_.reset(new NetworkStatsMap); + + // Regenerate stats from scratch. + for (DB::PrefixIterator iter(state_->db(), kNetworkQueueKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + int priority; + int64_t sequence; + if (DecodeNetworkQueueKey(key, &priority, &sequence)) { + (*stats_)[priority] += AdjustedCountForPriority(priority); + } + ServerOperation op; + if (op.ParseFromArray(value.data(), value.size())) { + for (int i = 0; i < op.stats_size(); ++i) { + (*stats_)[op.stats(i)] += AdjustedCountForPriority(op.stats(i)); + } + } + } + + LOG("network queue stats: %s, %.03f ms", *stats_, timer.Milliseconds()); +} + +ActivityHandle NetworkQueue::ProcessActivity( + const ActivityMetadata& m, const DBHandle& updates) { + ActivityHandle h = state_->activity_table()->LoadActivity( + m.activity_id().server_id(), updates); + const char* action = "update"; + if (!h.get()) { + h = state_->activity_table()->NewActivity(updates); + action = "new"; + } + + if (m.has_user_id()) { + // Fetch user information if user id is unknown. + state_->contact_manager()->MaybeQueueUser(m.user_id(), updates); + } + + h->Lock(); + h->MergeFrom(m); + + // Canonicalize viewpoint id to get a local id used to build mapping + // from viewpoint-id to list of activities. + state_->viewpoint_table()->CanonicalizeViewpointId( + h->mutable_viewpoint_id(), updates); + + h->Save(updates); + VLOG("photo: %s activity: %s", action, h->activity_id()); + return h; +} + +CommentHandle NetworkQueue::ProcessComment( + const CommentMetadata& m, const DBHandle& updates) { + CommentHandle h = state_->comment_table()->LoadComment( + m.comment_id().server_id(), updates); + const char* action = "update"; + if (!h.get()) { + h = state_->comment_table()->NewComment(updates); + action = "new"; + } + + h->Lock(); + h->MergeFrom(m); + + // Canonicalize viewpoint id to get a local id. + state_->viewpoint_table()->CanonicalizeViewpointId( + h->mutable_viewpoint_id(), updates); + + h->Save(updates); + + VLOG("photo: %s comment: %s", action, h->comment_id()); + return h; +} + +EpisodeHandle NetworkQueue::ProcessEpisode( + const EpisodeMetadata& m, bool recurse, const DBHandle& updates) { + EpisodeHandle h = state_->episode_table()->LoadEpisode(m.id(), updates); + const char* action = "update"; + if (!h.get()) { + h = state_->episode_table()->NewEpisode(updates); + action = "new"; + } + h->Lock(); + h->MergeFrom(m); + // Every episode loaded from server gets upload bit set to 0. + h->clear_upload_episode(); + + // Canonicalize viewpoint id to get a local id. + state_->viewpoint_table()->CanonicalizeViewpointId( + h->mutable_viewpoint_id(), updates); + + h->Save(updates); + VLOG("photo: %s episode: %s", action, h->id()); + if (recurse) { + // Synthesize an EpisodeSelection for this episode. + EpisodeSelection s; + s.set_episode_id(m.id().server_id()); + s.set_get_photos(true); + state_->episode_table()->Invalidate(s, updates); + } + return h; +} + +PhotoHandle NetworkQueue::ProcessPhoto( + const PhotoUpdate& u, EpisodeHandle* old_eh, const DBHandle& updates) { + const PhotoMetadata& m = u.metadata(); + return ProcessPhoto(state_->photo_table()->LoadPhoto(m.id(), updates), + u, old_eh, updates); +} + +// ProcessPhoto is called with server-supplied information about a photo in +// 'u'. This happens after upload_episode and query_episode. 'h' is the +// corresponding local photo, if any (for upload_episode there is always a +// local photo (but the local photo may not have a server_id yet), and for +// query_episodes the local photo is looked up by server_id). Processing a +// photo means merging the data in 'u' into a local photo (either 'h' or a +// newly-created photo). +PhotoHandle NetworkQueue::ProcessPhoto( + PhotoHandle h, const PhotoUpdate& u, + EpisodeHandle* old_eh, const DBHandle& updates) { + const PhotoMetadata& m = u.metadata(); + const char* action = "update"; + + if (!h.get()) { + // The server sent us a photo that we couldn't look up by server_id. Try + // to look it up by asset_key to see if we have a match. + for (int i = 0; !h.get() && i < m.asset_fingerprints_size(); i++) { + h = state_->photo_table()->LoadAssetPhoto(EncodeAssetKey("", m.asset_fingerprints(i)), updates); + if (!h.get()) { + continue; + } + // Found photo with matching asset key. + if (!state_->photo_table()->IsAssetPhotoEqual(*h, m)) { + // This is a pre-fingerprint photo that matched on url, but doesn't appear to actually be the same. + h.reset(NULL); + continue; + } else { + // There have been issues with duplicate photos created for + // the same asset key. Check here whether the photo we + // already have on disk has the same server id as the one + // which we're trying to add. If they're not the same, we + // have a duplicate. + + // NOTE(peter): There is a usage scenario which can cause this to + // occur. Existing viewfinder user has shared photo A. User gets new + // phone and re-installs viewfinder. After login, but before photo A is + // downloaded from the server, user decides to share photo A again. The + // second share will cause a server_id to be assigned to the photo + // which will prohibit the photo from being matched against the one on + // the server. This is a deemed to be too rare a scenario to fix and + // that effort should instead be put into adding functionality to + // quickly find and delete duplicates. + if (h->id().has_server_id() && m.id().has_server_id() && + h->id().server_id() != m.id().server_id()) { + LOG("duplicate photo with same asset key: %s != %s", *h, m); + h.reset(NULL); + continue; + } + } + } + } + + if (!h.get()) { + // We tried all the asset keys and still didn't find a match, so make a new photo. + h = state_->photo_table()->NewPhoto(updates); + action = "new"; + } + + h->Lock(); + + // We still fetch images for "label_hidden" images because they're + // hidden only in the feed, but not from the conversation to which + // they belong. + if (!m.label_unshared() && !m.label_removed()) { + if (!h->HasAssetUrl()) { + // No asset url which means the photo is not in the assets library. Check + // to see if we already have the various thumbnail/medium/full/original + // images. + if (!state_->photo_storage()->MaybeLinkServerId( + PhotoThumbnailFilename(h->id()), m.id().server_id(), + m.images().tn().md5(), updates)) { + if (u.has_tn_get_url()) { + h->SetURL("tn_get", u.tn_get_url(), updates); + } + h->set_download_thumbnail(true); + } + if (!state_->photo_storage()->MaybeLinkServerId( + PhotoMediumFilename(h->id()), m.id().server_id(), + m.images().med().md5(), updates)) { + if (u.has_med_get_url()) { + h->SetURL("med_get", u.med_get_url(), updates); + } + h->set_download_medium(true); + } + if (!state_->photo_storage()->MaybeLinkServerId( + PhotoFullFilename(h->id()), m.id().server_id(), + m.images().full().md5(), updates)) { + if (u.has_full_get_url()) { + h->SetURL("full_get", u.full_get_url(), updates); + } + h->set_download_full(true); + } + if (!state_->photo_storage()->MaybeLinkServerId( + PhotoOriginalFilename(h->id()), m.id().server_id(), + m.images().orig().md5(), updates)) { + if (u.has_orig_get_url()) { + h->SetURL("orig_get", u.orig_get_url(), updates); + } + // By default, we don't fetch the original image. It is fetched + // on occasions where it is necessary (e.g., if a resolution is + // required for display which exceeds full-resolution). + } + } else { + // The photo is stored in the asset library. Look for matching asset + // symlinks which indicate the photo has already been uploaded to the + // server. (under any of the possibly-multiple asset urls) + for (int i = 0; i < h->asset_keys_size(); i++) { + if (h->upload_thumbnail() && + state_->photo_storage()->HaveAssetSymlink( + PhotoThumbnailFilename(h->id()), m.id().server_id(), + h->asset_keys(i))) { + h->clear_upload_thumbnail(); + } + if (h->upload_medium() && + state_->photo_storage()->HaveAssetSymlink( + PhotoMediumFilename(h->id()), m.id().server_id(), + h->asset_keys(i))) { + h->clear_upload_medium(); + } + if (h->upload_full() && + state_->photo_storage()->HaveAssetSymlink( + PhotoFullFilename(h->id()), m.id().server_id(), + h->asset_keys(i))) { + h->clear_upload_full(); + } + if (h->upload_original() && + state_->photo_storage()->HaveAssetSymlink( + PhotoOriginalFilename(h->id()), m.id().server_id(), + h->asset_keys(i))) { + h->clear_upload_original(); + } + } + } + + if (h->upload_thumbnail() && u.has_tn_put_url()) { + h->SetURL("tn_put", u.tn_put_url(), updates); + } + if (h->upload_medium() && u.has_med_put_url()) { + h->SetURL("med_put", u.med_put_url(), updates); + } + if (h->upload_full() && u.has_full_put_url()) { + h->SetURL("full_put", u.full_put_url(), updates); + } + if (h->upload_original() && u.has_orig_put_url()) { + h->SetURL("orig_put", u.orig_put_url(), updates); + } + } + + // If the existing photo has a location & placemark and has been added to the + // placemark histogram, we need to remove it in case this update has modified + // location/placemark. This handles the following cases: + // - No change: restored to histogram on PhotoTable_Photo::Save() + // - Modified placemark: new info added to histogram on PhotoTable_Photo::Save() + // - Placemark/location deleted: removal from histogram is permanent + if (h->has_location() && h->has_placemark() && h->placemark_histogram()) { + h->clear_placemark_histogram(); + state_->placemark_histogram()->RemovePlacemark( + h->placemark(), h->location(), updates); + } + + if (old_eh && m.episode_id().has_server_id() && + h->episode_id().server_id() != m.episode_id().server_id()) { + EpisodeHandle eh = state_->episode_table()->LoadEpisode(h->episode_id(), updates); + if (eh.get() && eh->id().server_id() != m.episode_id().server_id()) { + // The server is changing the canonical episode for the photo. Most + // likely this is because the client state was reset and we've downloaded + // a photo that already existed in the asset library. We return to the + // caller the old episode handle so that it can remove the photo from the + // old episode. We can't remove the photo from the episode here as doing + // so could leave the photo unreferenced by an episode causing its images + // and assets to be deleted. + *old_eh = eh; + h->clear_episode_id(); + } + } + + h->clear_upload_metadata(); + h->clear_update_metadata(); + h->clear_label_hidden(); + h->clear_label_removed(); + h->clear_label_unshared(); + h->clear_error_upload_metadata(); + h->clear_error_timestamp(); + + // Construct a new PhotoMetadata that clears any field that should not be + // merged directly. + PhotoMetadata sanitized_metadata(m); + sanitized_metadata.clear_asset_keys(); + sanitized_metadata.clear_asset_fingerprints(); + h->MergeFrom(sanitized_metadata); + + DCHECK_EQ(m.asset_keys_size(), 0); + // If the server gave us any asset fingerprints, copy them to the local photo. + for (int i = 0; i < m.asset_fingerprints_size(); i++) { + h->AddAssetFingerprint(m.asset_fingerprints(i), true); + } + + LOG("photo: %s photo: %s (%s): %supload%s%s%s%s download%s%s%s%s", + action, h->id(), h->episode_id(), + h->update_metadata() ? "update:metadata " : "", + h->upload_thumbnail() ? ":thumbnail" : "", + h->upload_medium() ? ":medium" : "", + h->upload_full() ? ":full" : "", + h->upload_original() ? ":original" : "", + h->download_thumbnail() ? ":thumbnail" : "", + h->download_medium() ? ":medium" : "", + h->download_full() ? ":full" : "", + h->download_original() ? ":original" : ""); + + h->Save(updates); + return h; +} + +ViewpointHandle NetworkQueue::ProcessViewpoint( + const ViewpointMetadata& m, bool recurse, const DBHandle& updates) { + ViewpointHandle h = state_->viewpoint_table()->LoadViewpoint( + m.id().server_id(), updates); + const char* action = "update"; + if (!h.get()) { + h = state_->viewpoint_table()->NewViewpoint(updates); + action = "new"; + } + h->Lock(); + h->MergeFrom(m); + h->Save(updates); + VLOG("photo: %s viewpoint: %s", action, h->id()); + + if (recurse) { + // Synthesize a ViewpointSelection for this viewpoint. + ViewpointSelection s; + s.set_viewpoint_id(m.id().server_id()); + s.set_get_activities(true); + s.set_get_episodes(true); + s.set_get_followers(true); + s.set_get_comments(true); + state_->viewpoint_table()->Invalidate(s, updates); + } + return h; +} + +void NetworkQueue::MaybeQueueNetwork(int priority) { + MutexLock l(&queue_mu_); + if (!state_->is_registered()) { + // The user is not logged in. Clear any queued item(s). + queued_download_photo_.reset(NULL); + queued_remove_photos_.reset(NULL); + queued_update_viewpoint_.reset(NULL); + queued_upload_episode_.reset(NULL); + queued_upload_photo_.reset(NULL); + queued_upload_activity_.reset(NULL); + return; + } + if (queued_download_photo_.get() || + queued_remove_photos_.get() || + queued_update_viewpoint_.get() || + queued_upload_episode_.get() || + queued_upload_photo_.get() || + queued_upload_activity_.get()) { + // An item is already queued, do not change it because the network request + // might currently be in progress. + return; + } + if (queue_in_progress_) { + if (queue_start_time_ > 0) { + VLOG("photo: queue still in progress: %.03f ms", + 1000 * (WallTime_Now() - queue_start_time_)); + } + return; + } + + // The network queue schedules in what order to + // upload/download/remove/share/etc photos. The various MaybeQueue* methods + // take a ServerOperation and try and create a queued_* operation. When the + // queued operation is completed by the NetworkManager, the corresponding + // CommitQueued* method is called. Note that the CommitQueued* methods do not + // modify the queue. Instead, the code MaybeQueue* checks to see if the + // specified work has already been done. For example, if + // MaybeQueueUploadPhoto checks that some portion of the photo still needs to + // be uploaded. If the photo has been completely uploaded, that queue entry + // is moved and the loop below advances to the next one. + // + // This setup provides both robustness and the ability to queue the same + // operation at different priorities. For example, downloading of a photo + // might initially be queued at PRIORITY_DOWNLOAD_PHOTO. But if the user + // attempts to view that photo before it has been downloaded, another queue + // entry with priority PRIORITY_UI_THUMBNAIL will be created. + + DBHandle updates = state_->NewDBTransaction(); + for (ScopedPtr iter(NewIterator()); + !iter->done() && iter->priority() <= priority; + iter->Next()) { + while (!ShouldProcessPriority(iter->priority())) { + iter->SkipPriority(); + if (iter->done()) { + break; + } + } + if (iter->done()) { + break; + } + + const ServerOperation& op = iter->op(); + if (op.has_update_viewpoint()) { + if (MaybeQueueUpdateViewpoint(op, updates)) { + break; + } + } else if (op.has_upload_activity()) { + if (MaybeQueueUploadActivity(op, iter->priority(), updates)) { + break; + } + // TODO(pmattis): Quarantine the activity. + } else if (op.has_update_photo()) { + if (MaybeQueueUpdatePhoto(op, iter->priority(), updates)) { + break; + } + // TODO(pmattis): Quarantine the photo. + } else if (op.has_remove_photos()) { + if (MaybeQueueRemovePhotos( + op, iter->priority(), iter->sequence(), updates)) { + break; + } + } + // If we fall through to here, we were unable to queue the server + // operation. Remove it from the queue. + VLOG("dequeuing %d,%d (unable to process)", + iter->priority(), iter->sequence()); + Remove(iter->priority(), iter->sequence(), op, updates); + } + updates->Commit(); +} + +bool NetworkQueue::MaybeQueueUpdateViewpoint( + const ServerOperation& op, const DBHandle& updates) { + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint( + op.update_viewpoint(), updates); + if (!vh.get() || vh->label_error()) { + return false; + } + if (!vh->update_metadata() && + !vh->update_follower_metadata() && + !vh->update_remove() && + !vh->update_viewed_seq()) { + return false; + } + + ScopedPtr u(new UpdateViewpoint); + u->viewpoint = vh; + u->headers.CopyFrom(op.headers()); + + queued_update_viewpoint_.reset(u.release()); + return true; +} + +bool NetworkQueue::MaybeQueueUploadActivity( + const ServerOperation& op, int priority, const DBHandle& updates) { + ActivityHandle ah = state_->activity_table()->LoadActivity( + op.upload_activity(), updates); + if (!ah.get() || ah->label_error()) { + // Unable to find the queued activity or the activity is quarantined. + return false; + } + if (!ah->upload_activity()) { + // The queued activity no longer needs to be uploaded, presumably because + // it has already been uploaded. + return false; + } + if (!ah->has_share_new() && !ah->has_share_existing() && + !ah->has_add_followers() && !ah->has_post_comment() && + !ah->has_remove_followers() && !ah->has_save_photos() && + !ah->has_unshare()) { + // Huh, how did this activity get queued? + return false; + } + if (ah->has_share_new() && + ah->share_new().contacts_size() == 0) { + // A share_new with 0 contacts is invalid and the server will reject + // it. Mark the activity as provisional again so that the user can fix up + // the problem. Note that ConversationLayoutController no longer allows the + // creation of share_new activities with 0 contacts. + ah->Lock(); + ah->set_provisional(true); + ah->SaveAndUnlock(updates); + return false; + } + + ScopedPtr u(new UploadActivity); + if (ah->has_viewpoint_id()) { + u->viewpoint = state_->viewpoint_table()->LoadViewpoint(ah->viewpoint_id(), updates); + if (!u->viewpoint.get()) { + // Unable to find the viewpoint to share to. This shouldn't happen. + return false; + } + } + u->headers.CopyFrom(op.headers()); + u->activity = ah; + + if (ah->has_share_new() || ah->has_share_existing() || + ah->has_save_photos() || ah->has_unshare()) { + // Ensure that all of the photos in the share have been uploaded. + const ShareEpisodes* episodes = ah->GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + u->episodes.push_back(Episode()); + Episode* e = &u->episodes.back(); + const ActivityMetadata::Episode& episode = episodes->Get(i); + + e->episode = state_->episode_table()->LoadEpisode(episode.episode_id(), updates); + if (!e->episode.get()) { + // Unable to find episode. This shouldn't happen. + u->episodes.pop_back(); + continue; + } + + for (int j = 0; j < episode.photo_ids_size(); ++j) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(episode.photo_ids(j), updates); + if (!ph.get() || ph->label_error()) { + // Skip non-existent photos or photos with errors. + continue; + } + if (ph->upload_metadata() || ph->upload_thumbnail() || ph->upload_full()) { + if (MaybeQueueUploadPhoto(ph, priority, updates)) { + return true; + } + // Skip photos that cannot be uploaded. + continue; + } + e->photos.push_back(ph); + } + + // NOTE(peter): We check for the existence of the parent episode and its + // server id after we have ensured that all of the shared photos have + // been uploaded. This ensures that the parent episode will have had a + // server id assigned to it. + e->parent = state_->episode_table()->LoadEpisode(e->episode->parent_id(), updates); + if (!e->parent.get() || e->parent->id().server_id().empty()) { + // Unable to find parent episode or parent episode has no id. This shouldn't happen. + u->episodes.pop_back(); + continue; + } + + if (e->photos.empty()) { + u->episodes.pop_back(); + } + } + + // If this is a share_new activity, add contacts. Also, reset the + // cover photo to the first photo which has been successfully + // uploaded and verified. + if (ah->has_share_new()) { + for (int i = 0; i < ah->share_new().contacts_size(); ++i) { + u->contacts.push_back(ah->share_new().contacts(i)); + } + if (!u->episodes.empty() && !u->episodes.front().photos.empty()) { + u->viewpoint->Lock(); + u->viewpoint->mutable_cover_photo()->mutable_photo_id()->CopyFrom( + u->episodes.front().photos[0]->id()); + u->viewpoint->mutable_cover_photo()->mutable_episode_id()->CopyFrom( + u->episodes.front().episode->id()); + u->viewpoint->SaveAndUnlock(updates); + } + } + if (u->contacts.empty() && u->episodes.empty()) { + return false; + } + } else if (ah->has_add_followers()) { + for (int i = 0; i < ah->add_followers().contacts_size(); ++i) { + u->contacts.push_back(ah->add_followers().contacts(i)); + } + if (u->contacts.empty()) { + return false; + } + } else if (ah->has_post_comment()) { + u->comment = state_->comment_table()->LoadComment( + ah->post_comment().comment_id(), updates); + } else if (ah->has_remove_followers()) { + if (ah->remove_followers().user_ids_size() == 0) { + return false; + } + } + + queued_upload_activity_.reset(u.release()); + return true; +} + +bool NetworkQueue::MaybeQueueUpdatePhoto( + const ServerOperation& op, int priority, const DBHandle& updates) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(op.update_photo(), updates); + if (!ph.get() || ph->label_error()) { + // This shouldn't be possible. We never remove photo metadata and + // quarantined photos are removed from the queue. + return false; + } + if (ph->download_thumbnail() || ph->download_full() || ph->download_original()) { + if (MaybeQueueDownloadPhoto(ph, updates)) { + return true; + } + } + if (ph->update_metadata() || ph->upload_metadata() || + ph->upload_thumbnail() || ph->upload_medium() || + ph->upload_full() || ph->upload_original()) { + if (MaybeQueueUploadPhoto(ph, priority, updates)) { + return true; + } + } + ph->Lock(); + ph->SaveAndUnlock(updates); + return false; +} + +bool NetworkQueue::MaybeQueueRemovePhotos( + const ServerOperation& op, int priority, + int64_t sequence, const DBHandle& updates) { + const ServerOperation::RemovePhotos& r = op.remove_photos(); + ScopedPtr rp(new RemovePhotos); + rp->headers.CopyFrom(op.headers()); + rp->queue.set_priority(priority); + rp->queue.set_sequence(sequence); + + for (int i = 0; i < r.episodes_size(); ++i) { + rp->episodes.push_back(Episode()); + Episode* e = &rp->episodes.back(); + const ActivityMetadata::Episode& episode = r.episodes(i); + + e->episode = state_->episode_table()->LoadEpisode(episode.episode_id(), updates); + if (!e->episode.get()) { + // Unable to find episode. This shouldn't happen. + rp->episodes.pop_back(); + continue; + } + + for (int j = 0; j < episode.photo_ids_size(); ++j) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(episode.photo_ids(j), updates); + if (!ph.get() || ph->label_error()) { + // Skip non-existent photos or photos with errors. + continue; + } + if (ph->upload_metadata()) { + // Skip photos that haven't been uploaded. + continue; + } + e->photos.push_back(ph); + } + + if (e->photos.empty()) { + // No photos to remove. + rp->episodes.pop_back(); + } + } + + if (rp->episodes.empty()) { + // Nothing to do. Too many errors? + return false; + } + + queued_remove_photos_.reset(rp.release()); + return true; +} + +bool NetworkQueue::MaybeQueueDownloadPhoto( + const PhotoHandle& ph, const DBHandle& updates) { + ScopedPtr d(new DownloadPhoto); + d->photo = ph; + + DCHECK(DirExists(photo_tmp_dir_)) << " " << photo_tmp_dir_ << " doesn't exist"; + + if (d->photo->download_thumbnail()) { + d->type = THUMBNAIL; + d->path = JoinPath(photo_tmp_dir_, PhotoThumbnailFilename(d->photo->id())); + d->url = d->photo->GetUnexpiredURL("tn_get", updates); + } else if (d->photo->download_full()) { + d->type = FULL; + d->path = JoinPath(photo_tmp_dir_, PhotoThumbnailFilename(d->photo->id())); + d->url = d->photo->GetUnexpiredURL("full_get", updates); + } else if (d->photo->download_original()) { + // We should never be trying to download the original more than once. + DCHECK(!d->photo->error_download_original()); + d->type = ORIGINAL; + d->path = JoinPath(photo_tmp_dir_, PhotoThumbnailFilename(d->photo->id())); + d->url = d->photo->GetUnexpiredURL("orig_get", updates); + } else { + // Nothing left to do for this photo. + return false; + } + + VLOG("photo: %s: queueing download photo: %s", ph->id(), d->url); + + // NOTE(peter): We never download medium images. It is faster to download the + // full size image and resize it. + + d->episode = state_->episode_table()->GetEpisodeForPhoto(ph, updates); + if (!d->episode.get()) { + // We were unable to find an episode the photo was part of. + QuarantinePhoto(ph, "queue download: unable to find episode", updates); + return false; + } + + queued_download_photo_.reset(d.release()); + return true; +} + +bool NetworkQueue::MaybeQueueUploadPhoto( + const PhotoHandle& ph, int priority, const DBHandle& updates) { + if (!ph->shared() && !state_->CloudStorageEnabled()) { + // Cloud storage is disabled and the photo has not been shared. Stop + // processing the queue. + VLOG("photo: not queueing unshared photo (cloud storage disabled): %s", *ph); + return false; + } + + WallTimer timer; + queue_start_time_ = WallTime_Now(); + + if (ph->upload_metadata()) { + // The photo metadata needs to be uploaded. + if (!ph->episode_id().has_local_id()) { + // The photo doesn't have an associated episode, upload isn't possible. + QuarantinePhoto(ph, "upload: no episode id", updates); + return false; + } + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ph->episode_id(), updates); + if (!eh.get()) { + // Unable to find the photo's episode, upload isn't possible. + QuarantinePhoto(ph, "upload: unable to load episode", updates); + return false; + } + if (!MaybeQueueUploadEpisode(eh, updates)) { + // If the episode couldn't be uploaded because it has no photos, + // presumably this photo was removed. Quarantine the photo. + QuarantinePhoto(ph, "upload: unable to upload episode", updates); + return false; + } + return true; + } else if (ph->update_metadata()) { + return MaybeQueueUpdatePhotoMetadata(ph, updates); + } + + EpisodeHandle eh = state_->episode_table()->LoadEpisode(ph->episode_id(), updates); + if (!eh.get()) { + // Unable to find the photo's episode, upload isn't possible. + QuarantinePhoto(ph, "upload: unable to load episode", updates); + return false; + } + if (!eh->id().has_server_id()) { + // The episode doesn't have an associated server id, upload isn't possible. + QuarantinePhoto(ph, "upload: no episode server id", updates); + return false; + } + + UploadPhoto* u = new UploadPhoto; + u->episode = eh; + u->photo = ph; + int size = 0; + + if (ph->upload_thumbnail()) { + u->type = THUMBNAIL; + u->url = u->photo->GetUnexpiredURL("tn_put", updates); + size = kThumbnailSize; + } else if (ph->upload_full()) { + u->type = FULL; + u->url = u->photo->GetUnexpiredURL("full_put", updates); + size = kFullSize; + } else if (ph->upload_medium()) { + u->type = MEDIUM; + u->url = u->photo->GetUnexpiredURL("med_put", updates); + size = kMediumSize; + } else if (ph->upload_original()) { + if (!state_->store_originals()) { + // If cloud storage of originals is turned off, stop processing the queue + // when we first see an operation for storing an original. + delete u; + if (priority != PRIORITY_UPLOAD_PHOTO_ORIGINAL) { + // Enforce PRIORITY_UPLOAD_PHOTO_ORIGINAL at this point in order for + // NetworkQueue::Empty() to work properly. Returning false will cause + // the photo to be saved. + return false; + } + return true; + } + u->type = ORIGINAL; + u->url = u->photo->GetUnexpiredURL("orig_put", updates); + size = kOriginalSize; + } else { + delete u; + return false; + } + + queue_in_progress_ = true; + + const string filename = PhotoFilename(u->photo->id(), size); + u->path = JoinPath(state_->photo_dir(), filename); + + const Callback done = [this, u, size, timer](bool success) { + if (success) { + if (size == kThumbnailSize) { + u->md5 = u->photo->images().tn().md5(); + } else if (size == kMediumSize) { + u->md5 = u->photo->images().med().md5(); + } else if (size == kFullSize) { + u->md5 = u->photo->images().full().md5(); + } else if (size == kOriginalSize) { + u->md5 = u->photo->images().orig().md5(); + } else { + CHECK(false); + } + CHECK_EQ(32, u->md5.size()); + } + + MutexLock l(&queue_mu_); + queued_upload_photo_.reset(u); + queue_in_progress_ = false; + queue_start_time_ = 0; + + // The queued upload is ready to go. Kick the NetworkManager. + VLOG("photo: queued upload photo: %s (%s): %.03f ms", + u->photo->id(), PhotoSizeSuffix(size), timer.Milliseconds()); + state_->async()->dispatch_main([this] { + // Queuing the upload photo might have failed, but we want to go + // through the same CommitQueueUploadPhoto() path and it is not + // thread-safe to call CommitQueuedUploadPhoto() directly. + state_->net_manager()->Dispatch(); + }); + }; + + if (state_->photo_storage()->Exists(filename) && + u->photo->images().has_tn() && + u->photo->images().has_med() && + u->photo->images().has_full() && + u->photo->images().has_orig()) { + state_->async()->dispatch_low_priority([done] { + done(true); + }); + } else { + u->photo->clear_images(); + state_->LoadViewfinderImages(u->photo->id().local_id(), u->photo->db(), done); + } + return true; +} + +bool NetworkQueue::MaybeQueueUpdatePhotoMetadata( + const PhotoHandle& ph, const DBHandle& updates) { + if (!ph->update_metadata()) { + return false; + } + + ScopedPtr u(new UpdatePhoto); + u->headers.set_op_id(state_->NewLocalOperationId()); + u->headers.set_op_timestamp(WallTime_Now()); + u->photo = ph; + + queued_update_photo_.reset(u.release()); + return true; +} + +bool NetworkQueue::MaybeQueueUploadEpisode( + const EpisodeHandle& eh, const DBHandle& updates) { + // NOTE: be careful if adding another "return false" case to this + // code, as MaybeQueueUploadPhoto will quarantine the photo. + queue_start_time_ = WallTime_Now(); + + ScopedPtr u(new UploadEpisode); + u->headers.set_op_id(state_->NewLocalOperationId()); + u->headers.set_op_timestamp(WallTime_Now()); + u->episode = eh; + + vector photo_ids; + u->episode->ListPhotos(&photo_ids); + for (int i = 0; i < photo_ids.size(); ++i) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_ids[i], updates); + if (!ph.get()) { + continue; + } + if (!ph->upload_metadata()) { + continue; + } + u->photos.push_back(ph); + if (u->photos.size() >= kMaxPhotosPerUpload) { + break; + } + } + + if (u->photos.empty()) { + // Nothing left to do for this episode. + return false; + } + + if (!u->episode->id().has_server_id()) { + // The episode was created before we knew our device id. + u->episode->Lock(); + CHECK(u->episode->MaybeSetServerId()); + u->episode->SaveAndUnlock(updates); + } + + queue_in_progress_ = true; + MaybeLoadImages(u.release(), 0); + return true; +} + +void NetworkQueue::MaybeReverseGeocode(UploadEpisode* u, int index) { + for (; index < u->photos.size(); ++index) { + const PhotoHandle& ph = u->photos[index]; + if (state_->photo_table()->ReverseGeocode( + ph->id().local_id(), [this, u, index](bool) { + MaybeReverseGeocode(u, index + 1); + })) { + return; + } + } + + // Note that we intentionally do not call dispatch_main() here as we want the + // stack to unwind and locks to be released before grabbing queue_mu_. + state_->async()->dispatch_main_async([this, u] { + { + MutexLock l(&queue_mu_); + + // All of the photos could have been removed (and presumably quarantined) + // during the upload queueing process. + const int64_t device_id = state_->device_id(); + if (device_id && !u->photos.empty()) { + DBHandle updates = state_->NewDBTransaction(); + for (int i = 0; i < u->photos.size(); ++i) { + const PhotoHandle& ph = u->photos[i]; + CHECK(ph->images().tn().has_md5()); + CHECK(ph->images().med().has_md5()); + CHECK(ph->images().full().has_md5()); + CHECK(ph->images().orig().has_md5()); + if (!ph->id().has_server_id()) { + // The photo was created before we knew our device id. + ph->Lock(); + CHECK(ph->MaybeSetServerId()); + ph->SaveAndUnlock(updates); + } + } + updates->Commit(); + + // The queued upload is ready to go. + const EpisodeHandle& eh = u->episode; + VLOG("photo: queued upload episode: %s: %.03f ms", + eh->id(), 1000 * (WallTime_Now() - queue_start_time_)); + queued_upload_episode_.reset(u); + } else { + delete u; + } + + queue_in_progress_ = false; + queue_start_time_ = 0; + } + + // Kick the NetworkManager. Even if no upload was queued, this will run the + // dispatch loop and cause another upload queue to take place. + state_->net_manager()->Dispatch(); + }); +} + +void NetworkQueue::MaybeLoadImages(UploadEpisode* u, int index) { + for (; index < u->photos.size(); ++index) { + PhotoHandle ph = u->photos[index]; + // Force the image data to be recomputed from scratch. + ph->clear_images(); + + const Callback done = [this, ph, u, index](bool success) { + if (success) { + // In the process of loading the photo data the photo might have been + // moved to a different episode. Don't upload it with the current + // episode if that occurred. + success = (ph->episode_id().local_id() == u->episode->id().local_id()); + } + int next_index = index + 1; + if (!success) { + // Remove the photo from the UploadEpisode. + u->photos.erase(u->photos.begin() + index); + --next_index; + } + MaybeLoadImages(u, next_index); + }; + + state_->LoadViewfinderImages(ph->id().local_id(), ph->db(), done); + return; + } + + dispatch_main([this, u] { + MaybeReverseGeocode(u, 0); + }); +} + +void NetworkQueue::QuarantinePhoto(PhotoHandle p, const string& reason) { + DBHandle updates = state_->NewDBTransaction(); + QuarantinePhoto(p, reason, updates); + updates->Commit(); +} + +void NetworkQueue::QuarantinePhoto( + PhotoHandle p, const string& reason, const DBHandle& updates) { + p->Lock(); + p->Quarantine(reason, updates); + p->SaveAndUnlock(updates); +} + +void NetworkQueue::UpdateViewpointError(ViewpointHandle vh) { + LOG("photo: quarantining viewpoint: %s", vh->id()); + DBHandle updates = state_->NewDBTransaction(); + vh->Lock(); + vh->set_label_error(true); + vh->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::UpdatePhotoError(PhotoHandle p) { + p->Lock(); + if (p->error_update_metadata()) { + // We had previously tried to upload this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "update: metadata"); + return; + } + p->set_error_update_metadata(true); + DBHandle updates = state_->NewDBTransaction(); + p->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::UploadPhotoError(PhotoHandle p, int types) { + p->Lock(); + if (types & THUMBNAIL) { + if (p->error_upload_thumbnail()) { + // We had previously tried to upload this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "upload: thumbnail"); + return; + } + p->set_error_upload_thumbnail(true); + } + if (types & MEDIUM) { + if (p->error_upload_medium()) { + // We had previously tried to upload this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "upload: medium"); + return; + } + p->set_error_upload_medium(true); + } + if (types & FULL) { + if (p->error_upload_full()) { + // We had previously tried to upload this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "upload: full"); + return; + } + p->set_error_upload_full(true); + } + if (types & ORIGINAL) { + if (p->error_upload_original()) { + // We had previously tried to upload this photo and encountered an + // error. We do not quarantine if we're unable to upload the original. + p->Unlock(); + return; + } + // Failure to upload the original is expected if upload_originals is turned + // on well after the original photo was added to the library. We do not + // quarantine such photos, but simply clear the upload_original bit. + p->clear_upload_original(); + p->set_error_upload_original(true); + DBHandle updates = state_->NewDBTransaction(); + p->SaveAndUnlock(updates); + updates->Commit(); + return; + } + + // Reset the photo state machine. The error_upload_* bits will be cleared + // when the photo is uploaded. + p->set_upload_thumbnail(true); + p->set_upload_medium(true); + p->set_upload_full(true); + p->set_upload_original(true); + + { + // Re-add the photo to an episode. + DBHandle updates = state_->NewDBTransaction(); + EpisodeHandle e = state_->episode_table()->LoadEpisode(p->episode_id(), updates); + state_->episode_table()->AddPhotoToEpisode(p, updates); + if (e.get() && e->id().local_id() != p->episode_id().local_id()) { + // The photo's episode changed. Remove it from the old episode. + e->Lock(); + e->RemovePhoto(p->id().local_id()); + e->SaveAndUnlock(updates); + } + updates->Commit(); + } + + DBHandle updates = state_->NewDBTransaction(); + p->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::DownloadPhotoError(PhotoHandle p, int types) { + if (!p->id().has_server_id()) { + // How on earth were we trying to download a photo without a corresponding + // server id. + QuarantinePhoto(p, "download: no server id"); + return; + } + if (!p->has_episode_id()) { + // A photo without an episode. Quarantine. + QuarantinePhoto(p, "download: no episode id"); + return; + } + + p->Lock(); + + // Reset the photo state machine. Download the metadata and images again. The + // error_download_* bits will be cleared when the image is successfully + // downloaded. + if (types & THUMBNAIL) { + if (p->error_download_thumbnail()) { + // We had previously tried to download this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "download: thumbnail"); + return; + } + p->set_error_download_thumbnail(true); + } + if (types & MEDIUM) { + if (p->error_download_medium()) { + // We had previously tried to download this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "download: medium"); + return; + } + p->set_error_download_medium(true); + } + if (types & FULL) { + if (p->error_download_full()) { + // We had previously tried to download this photo and encountered an + // error. Quarantine the photo. + p->Unlock(); + QuarantinePhoto(p, "download: full"); + return; + } + p->set_error_download_full(true); + } + if (types & ORIGINAL) { + // Never quarantine a photo because the original isn't available. + DCHECK(!p->error_download_original()); + // Set the download error and unset the intention to download it. + // The assumption is that the photo was never uplaoded and isn't + // ever going to be available. This isn't necessarily an error. + p->set_error_download_original(true); + p->set_download_original(false); + } + + DBHandle updates = state_->NewDBTransaction(); + + // If thumbnail or full resolution is missing, synthesize an + // EpisodeSelection for this episode in order to download the photo + // metadata again. + if (p->error_download_thumbnail() || p->error_download_full()) { + EpisodeHandle found_episode = + state_->episode_table()->GetEpisodeForPhoto(p, updates); + + if (!found_episode.get()) { + // Couldn't find the photo's episode. Quarantine. + p->Unlock(); + QuarantinePhoto(p, "download: unable to find episode"); + return; + } + + EpisodeSelection s; + s.set_episode_id(found_episode->id().server_id()); + s.set_get_photos(true); + state_->episode_table()->Invalidate(s, updates); + } + + p->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::UploadActivityError(ActivityHandle ah) { + LOG("photo: quarantining activity: %s", ah->activity_id()); + DBHandle updates = state_->NewDBTransaction(); + ah->Lock(); + ah->set_label_error(true); + ah->SaveAndUnlock(updates); + updates->Commit(); +} + +void NetworkQueue::NotifyDownload(int64_t photo_id, int types) { + MutexLock l(&download_callback_mu_); + DownloadCallbackSet* callbacks = + FindOrNull(&download_callback_map_, photo_id); + if (!callbacks) { + return; + } + callbacks->Run(types); + if (callbacks->empty()) { + download_callback_map_.erase(photo_id); + delete callbacks; + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/NetworkQueue.h b/clients/shared/NetworkQueue.h new file mode 100644 index 0000000..0764edb --- /dev/null +++ b/clients/shared/NetworkQueue.h @@ -0,0 +1,350 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_NETWORK_QUEUE_H +#define VIEWFINDER_NETWORK_QUEUE_H + +#import +#import "ActivityTable.h" +#import "CommentTable.h" +#import "DB.h" +#import "EpisodeTable.h" +#import "Mutex.h" +#import "PhotoTable.h" +#import "Server.pb.h" +#import "ViewpointTable.h" + +class AppState; +class PhotoUpdate; + +// These priorities are persisted to disk, so be careful when changing them. +enum { + // Prioritize operations performed explicitly by the user or required by the + // UI. For example, prioritize the retrieval of thumbnail images that need to + // be displayed on screen. + PRIORITY_UI_THUMBNAIL = 10, + PRIORITY_UI_FULL = 20, + PRIORITY_UI_ACTIVITY = 50, + PRIORITY_UI_UPLOAD_PHOTO = 70, + PRIORITY_UI_ORIGINAL = 100, + // Used by NetworkManager to clear all preceding priority levels. + PRIORITY_UI_MAX = 101, + // The priority band for viewpoint updates, such as updating the viewed + // sequence number. + PRIORITY_UPDATE_VIEWPOINT = 300, + // The priority bands for uploading photos that are used by the client for + // display. + PRIORITY_UPLOAD_PHOTO = 400, + PRIORITY_UPLOAD_PHOTO_MEDIUM = 500, + // The priority band for downloading photos that are not visible on the + // screen. + PRIORITY_DOWNLOAD_PHOTO = 550, + // The priority band for uploading original images. + PRIORITY_UPLOAD_PHOTO_ORIGINAL = 600, + PRIORITY_MAX = 1000, +}; + +// The NetworkQueue maintains a map from: +// , -> +// -> , +// +// Both priorities and sequence numbers are sorted such that lower values come +// before higher values. The sequence number is internally allocated and +// ensures FIFO for operations with the same priority. +class NetworkQueue { + typedef CallbackSet1 DownloadCallbackSet; + typedef std::unordered_map< + int64_t, DownloadCallbackSet*> DownloadCallbackMap; + + public: + class Iterator { + public: + Iterator(leveldb::Iterator* iter); + ~Iterator(); + + // Advance to the next queued operation. + void Next(); + + // Skip to the next priority band containing a queued operation. + void SkipPriority(); + + const ServerOperation& op() const { return op_; } + bool done() const { return done_; } + int priority() const { return priority_; } + int64_t sequence() const { return sequence_; } + + private: + bool UpdateState(); + + private: + ScopedPtr iter_; + bool done_; + int priority_; + int64_t sequence_; + ServerOperation op_; + }; + + enum PhotoType { + THUMBNAIL = PhotoMetadata::THUMBNAIL, + MEDIUM = PhotoMetadata::MEDIUM, + FULL = PhotoMetadata::FULL, + ORIGINAL = PhotoMetadata::ORIGINAL, + }; + + struct Episode { + EpisodeHandle parent; + EpisodeHandle episode; + vector photos; + }; + + struct DownloadPhoto { + EpisodeHandle episode; + PhotoHandle photo; + PhotoType type; + string path; + string url; + }; + + struct RemovePhotos { + OpHeaders headers; + QueueMetadata queue; + vector episodes; + }; + + struct UpdatePhoto { + OpHeaders headers; + PhotoHandle photo; + }; + + struct UpdateViewpoint { + OpHeaders headers; + ViewpointHandle viewpoint; + }; + + // UploadActivity handles a locally-created activity which still must be sent + // to the server via share_new, share_existing, add_followers, post_comment + // or unshare operations. + struct UploadActivity { + OpHeaders headers; + ActivityHandle activity; + ViewpointHandle viewpoint; + // For shares. + vector episodes; + // For shares & add_followers. + vector contacts; + // For post comment. + CommentHandle comment; + }; + + struct UploadEpisode { + OpHeaders headers; + EpisodeHandle episode; + vector photos; + }; + + struct UploadPhoto { + EpisodeHandle episode; + PhotoHandle photo; + PhotoType type; + string url; + string path; + string md5; + }; + + public: + NetworkQueue(AppState* state); + ~NetworkQueue(); + + // Add an operation with the specified priority to the queue. Note that + // nothing in the NetworkQueue prohibits the same operation from being + // enqueued multiple times at the same or different priorities. Returns the + // sequence number of the newly added entry. + int64_t Add(int priority, const ServerOperation& op, const DBHandle& updates); + + // Remove the operation with the specified priority and sequence number from + // the queue. This should only be called once the operation has completed and + // before retrieving the next operation from the queue. + void Remove(int priority, int64_t sequence, const DBHandle& updates); + void Remove(int priority, int64_t sequence, + const ServerOperation& op, const DBHandle& updates); + + // Queue the specified photo, possibly moving it from its current position in + // the queue to a new position. Returns true iff the photo was modified. + bool QueuePhoto(const PhotoHandle& ph, const DBHandle& updates); + + // Dequeue the specified photo. Returns true iff the photo was modified. + bool DequeuePhoto(const PhotoHandle& ph, const DBHandle& updates); + + // Queue the specified activity. Returns true iff the activity was modified. + bool QueueActivity(const ActivityHandle& ah, const DBHandle& updates); + + // Dequeue the specified activity. Returns true iff the activity was + // modified. + bool DequeueActivity(const ActivityHandle& ah, const DBHandle& updates); + + // Queue the specified viewpoint. Returns true iff the viewpoint was modified. + bool QueueViewpoint(const ViewpointHandle& vh, const DBHandle& updates); + + // Dequeue the specified viewpoint. Returns true iff the viewpoint was + // modified. + bool DequeueViewpoint(const ViewpointHandle& vh, const DBHandle& updates); + + // Returns a new Iterator object for iterating over the queued + // operations. The caller is responsible for deleting the iterator. + Iterator* NewIterator(); + + // Returns true iff nothing is queued. + bool Empty(); + // Returns the priority of the item on the top of the queue, or -1 if nothing + // is queued. + int TopPriority(); + + /// Returns the (adjusted) number of pending network operations. + int GetNetworkCount(); + int GetDownloadCount(); + int GetUploadCount(); + + // Returns true iff we should process the given priority band given the + // network (wifi vs 3g/lte) and other settings. + bool ShouldProcessPriority(int priority) const; + // Returns true iff the given priority band corresponds to a download. + bool IsDownloadPriority(int priority) const; + + // Commit queued requests. + void CommitQueuedDownloadPhoto(const string& md5, bool retry); + void CommitQueuedRemovePhotos(bool error); + void CommitQueuedUpdatePhoto(bool error); + enum UpdateViewpointType { + UPDATE_VIEWPOINT_METADATA, + UPDATE_VIEWPOINT_FOLLOWER_METADATA, + UPDATE_VIEWPOINT_REMOVE, + UPDATE_VIEWPOINT_VIEWED_SEQ, + }; + void CommitQueuedUpdateViewpoint(UpdateViewpointType type, bool error); + void CommitQueuedUploadEpisode(const UploadEpisodeResponse& r, int status); + void CommitQueuedUploadPhoto(bool error); + void CommitQueuedUploadActivity(bool error); + + // Process server responses. + void ProcessQueryEpisodes( + const QueryEpisodesResponse& r, const vector& v, + const DBHandle& updates); + void ProcessQueryFollowed( + const QueryFollowedResponse& r, const DBHandle& updates); + void ProcessQueryNotifications( + const QueryNotificationsResponse& r, const DBHandle& updates); + void ProcessQueryViewpoints( + const QueryViewpointsResponse& r, const vector& v, + const DBHandle& updates); + + // Wait for the specified photo to be downloaded, invoking "done" when the + // photo has been downloaded or an error has occurred. It is the callers + // responsibility to ensure that the specified photo has been queued for + // download. + void WaitForDownload( + int64_t photo_id, PhotoType desired_type, Callback done); + + // Returns a map containing counts of enqueued operations by priority. + // Should be converted to an integer (with ceil()) before display. + // Fractional values are used to compensate for the fact that one user action + // may result in multiple queued operations (e.g. uploading metadata, + // thumbnail, medium, full sizes are 0.25 each so the counter goes up by one + // for each photo taken). + typedef std::map NetworkStatsMap; + NetworkStatsMap stats(); + + const DownloadPhoto* queued_download_photo() const { + return queued_download_photo_.get(); + } + const RemovePhotos* queued_remove_photos() const { + return queued_remove_photos_.get(); + } + const UpdatePhoto* queued_update_photo() const { + return queued_update_photo_.get(); + } + const UpdateViewpoint* queued_update_viewpoint() const { + return queued_update_viewpoint_.get(); + } + const UploadEpisode* queued_upload_episode() const { + return queued_upload_episode_.get(); + } + const UploadPhoto* queued_upload_photo() const { + return queued_upload_photo_.get(); + } + const UploadActivity* queued_upload_activity() const { + return queued_upload_activity_.get(); + } + + private: + void UpdateStatsLocked(int priority, const ServerOperation& op, bool addition); + + void EnsureInitLocked(); + void EnsureStatsInitLocked(); + + ActivityHandle ProcessActivity(const ActivityMetadata& m, const DBHandle& updates); + CommentHandle ProcessComment(const CommentMetadata& m, const DBHandle& updates); + EpisodeHandle ProcessEpisode(const EpisodeMetadata& m, + bool recurse, const DBHandle& updates); + PhotoHandle ProcessPhoto(const PhotoUpdate& u, EpisodeHandle* old_eh, + const DBHandle& updates); + PhotoHandle ProcessPhoto(PhotoHandle h, const PhotoUpdate& u, + EpisodeHandle* old_eh, const DBHandle& updates); + ViewpointHandle ProcessViewpoint(const ViewpointMetadata& m, + bool recurse, const DBHandle& updates); + + void MaybeQueueNetwork(int priority); + bool MaybeQueueUploadActivity(const ServerOperation& op, int priority, + const DBHandle& updates); + bool MaybeQueueUpdatePhoto(const ServerOperation& op, int priority, + const DBHandle& updates); + bool MaybeQueueRemovePhotos(const ServerOperation& op, int priority, + int64_t sequence, const DBHandle& updates); + bool MaybeQueueDownloadPhoto(const PhotoHandle& ph, const DBHandle& updates); + bool MaybeQueueUpdateViewpoint(const ServerOperation& op, const DBHandle& updates); + bool MaybeQueueUploadPhoto(const PhotoHandle& ph, int priority, + const DBHandle& updates); + bool MaybeQueueUpdatePhotoMetadata(const PhotoHandle& ph, const DBHandle& updates); + bool MaybeQueueUploadEpisode(const EpisodeHandle& eh, const DBHandle& updates); + void MaybeReverseGeocode(UploadEpisode* u, int index); + void MaybeLoadImages(UploadEpisode* u, int index); + + void QuarantinePhoto(PhotoHandle p, const string& reason); + void QuarantinePhoto(PhotoHandle p, const string& reason, + const DBHandle& updates); + + void UpdateViewpointError(ViewpointHandle vh); + void UpdatePhotoError(PhotoHandle p); + void UploadPhotoError(PhotoHandle p, int types); + void DownloadPhotoError(PhotoHandle p, int types); + void UploadActivityError(ActivityHandle ah); + + void NotifyDownload(int64_t photo_id, int types); + + private: + AppState* state_; + int64_t next_sequence_; + mutable Mutex mu_; + ScopedPtr stats_; + Mutex queue_mu_; + bool queue_in_progress_; + WallTime queue_start_time_; + ScopedPtr queued_download_photo_; + ScopedPtr queued_remove_photos_; + ScopedPtr queued_update_photo_; + ScopedPtr queued_update_viewpoint_; + ScopedPtr queued_upload_episode_; + ScopedPtr queued_upload_photo_; + ScopedPtr queued_upload_activity_; + const string photo_tmp_dir_; + Mutex download_callback_mu_; + DownloadCallbackMap download_callback_map_; +}; + +string EncodeNetworkQueueKey(int priority, int64_t sequence); +bool DecodeNetworkQueueKey(Slice key, int* priority, int64_t* sequence); + +#endif // VIEWFINDER_NETWORK_QUEUE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/NotificationManager.cc b/clients/shared/NotificationManager.cc new file mode 100644 index 0000000..3affa7d --- /dev/null +++ b/clients/shared/NotificationManager.cc @@ -0,0 +1,141 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import "AppState.h" +#import "DB.h" +#import "Logging.h" +#import "NetworkManager.h" +#import "NotificationManager.h" +#import "Server.pb.h" +#import "StringUtils.h" + +namespace { + +const string kNotificationSelectionKey = + DBFormat::metadata_key("notification_selection"); + +const DBRegisterKeyIntrospect kNotificationSelectionKeyIntrospect( + kNotificationSelectionKey, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +} // namespace + + +NotificationManager::NotificationManager(AppState* state) + : state_(state) { + CommonInit(); +} + +NotificationManager::~NotificationManager() { +} + +void NotificationManager::RemoteNotification(const string& message) { + DBHandle updates = state_->NewDBTransaction(); + Invalidate(updates); + updates->Commit(); + + state_->net_manager()->ResetBackoff(); + state_->net_manager()->Dispatch(); +} + +void NotificationManager::ProcessQueryNotifications(const QueryNotificationsResponse& p, + const NotificationSelection& ns, + bool process, const DBHandle& updates) { + last_query_time_ = WallTime_Now(); + + Validate(ns, updates); + + // A "nuclear" invalidation happens when the last key is cleared in + // the notification selection. This can happen from an invalidate + // all request or a missing notification sequence number. + if (!ns.query_done() && ns.last_key().empty()) { + LOG("notification: initiating nuclear invalidation of all assets"); + nuclear_invalidations_.Run(updates); + } else if (process) { + process_notifications_.Run(p, updates); + } +} + +void NotificationManager::Validate(const NotificationSelection& ns, const DBHandle& updates) { + NotificationSelection existing; + updates->GetProto(kNotificationSelectionKey, &existing); + if (ns.has_last_key()) { + existing.set_last_key(ns.last_key()); + } + if (ns.has_query_done()) { + existing.set_query_done(ns.query_done()); + } + + // Track the low-water mark for notifications which arrive with a + // min-required-version which this client doesn't understand. + if (ns.has_max_min_required_version() && + ns.max_min_required_version() > existing.max_min_required_version()) { + existing.set_max_min_required_version(ns.max_min_required_version()); + } + if (ns.has_low_water_notification_id() && + ns.low_water_notification_id() <= existing.low_water_notification_id()) { + existing.set_low_water_notification_id(ns.low_water_notification_id()); + } + + // Handle low water notification id and max min_required version in + // case of a nuclear invalidation. + if (!ns.query_done() && ns.last_key().empty()) { + // If the server is insisting on a min required version this + // client doesn't understand, the low-water mark on a nuclear + // invalidation must reset to the very beginning. + if (existing.max_min_required_version() > state_->protocol_version()) { + existing.set_low_water_notification_id(0); + } + } + updates->PutProto(kNotificationSelectionKey, existing); +} + +void NotificationManager::Invalidate(const DBHandle& updates) { + NotificationSelection existing; + updates->GetProto(kNotificationSelectionKey, &existing); + existing.clear_query_done(); + updates->PutProto(kNotificationSelectionKey, existing); +} + +bool NotificationManager::GetInvalidation(NotificationSelection* ns) { + if (!state_->db()->GetProto(kNotificationSelectionKey, ns)) { + LOG("notification: WARNING, notification selection missing"); + CommonInit(); + state_->db()->GetProto(kNotificationSelectionKey, ns); + } + // If "query_done" is false, we're ready to query invalidations. + return !ns->query_done(); +} + +void NotificationManager::CommonInit() { + last_query_time_ = 0; + + // Query the notification selection in case this client is finally + // new enough to understand past notifications. If so, update last key + // and clear the max_min_required_version and low_water_notification_id. + NotificationSelection ns; + if (state_->db()->GetProto(kNotificationSelectionKey, &ns)) { + if (ns.has_max_min_required_version() && + ns.max_min_required_version() <= state_->protocol_version()) { + if (!ns.low_water_notification_id()) { + ns.set_last_key(""); + } else { + ns.set_last_key(ToString(ns.low_water_notification_id())); + } + ns.clear_low_water_notification_id(); + ns.clear_max_min_required_version(); + } + ns.clear_query_done(); + state_->db()->PutProto(kNotificationSelectionKey, ns); + } else { + ns.Clear(); + ns.clear_query_done(); + ns.set_last_key(""); + state_->db()->PutProto(kNotificationSelectionKey, ns); + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/NotificationManager.h b/clients/shared/NotificationManager.h new file mode 100644 index 0000000..302148f --- /dev/null +++ b/clients/shared/NotificationManager.h @@ -0,0 +1,78 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_NOTIFICATION_MANAGER_H +#define VIEWFINDER_NOTIFICATION_MANAGER_H + +#import "Callback.h" +#import "DB.h" +#import "InvalidateMetadata.pb.h" +#import "ScopedPtr.h" +#import "WallTime.h" + +class AppState; +class NotificationSelection; +class QueryNotificationsResponse; + +class NotificationManager { + public: + NotificationManager(AppState* state); + ~NotificationManager(); + + // Application received a remote APNs notification message. + void RemoteNotification(const string& message); + + // Processes the results of call to query_notifications. The last + // query notification key is updated, and if necessary, the + // low-water mark for notifications with min_required_version set + // too high for this client to understand. Invokes the callbacks in + // the ProcessNotificationsCallback set under normal conditions. If + // a nuclear, all-out invalidation is specified, or if a gap in the + // notification sequence is detected, internal query state is fully + // reset and the callbacks in NuclearInvalidationCallback are invoked. + // + // If "process" is false, the notification process callbacks are not + // invoked. This is the case when querying notifications to find the + // high water mark at the start of rebuilding full asset state. + void ProcessQueryNotifications(const QueryNotificationsResponse& p, + const NotificationSelection& cs, + bool process, const DBHandle& updates); + + // Validates queried notifications. + void Validate(const NotificationSelection& s, const DBHandle& updates); + + // Invalidates notification selection so that new notifications are + // queried. If applicable, augments or creates the + // NotificationSelection which indicates which notifications are + // considered invalid due to a server response with + // min_required_version too high for the client to understand. + void Invalidate(const DBHandle& updates); + + // Gets the current notification selection. Returns true if the + // notification selection is invalidated and requires re-querying; + // false otherwise. + bool GetInvalidation(NotificationSelection* cs); + + // Callback set for processing query notifications. + // Used by PhotoManager and ContactManager. + typedef CallbackSet2 ProcessNotificationsCallback; + ProcessNotificationsCallback* process_notifications() { return &process_notifications_; } + + // Callback set for resetting query state. + // Used by PhotoManager, ContactManager, & NetworkManager. + typedef CallbackSet1 NuclearInvalidationCallback; + NuclearInvalidationCallback* nuclear_invalidations() { return &nuclear_invalidations_; } + + private: + void CommonInit(); + + private: + AppState* state_; + WallTime last_query_time_; + string query_notifications_last_key_; + NotificationSelection notification_selection_; + ProcessNotificationsCallback process_notifications_; + NuclearInvalidationCallback nuclear_invalidations_; +}; + +#endif // VIEWFINDER_NOTIFICATION_MANAGER_H diff --git a/clients/shared/PathUtils.android.cc b/clients/shared/PathUtils.android.cc new file mode 100644 index 0000000..24a5316 --- /dev/null +++ b/clients/shared/PathUtils.android.cc @@ -0,0 +1,68 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault. + +#import "FileUtils.h" +#import "Format.h" +#import "Logging.h" +#import "PathUtils.h" +#import "StringUtils.h" + +namespace { + +string TmpPath() { + return "tmp"; +} + +class AppDir : public string { + public: + AppDir(const string& dir) { + DirCreate(dir); + DirCreate(JoinPath(dir, LibraryPath())); + DirCreate(JoinPath(dir, TmpPath())); + assign(dir); + } +}; + +AppDir* kAppDir = NULL; + +} // namespace + +void InitApplicationPath(const string& dir) { + if (kAppDir == NULL) { + kAppDir = new AppDir(dir); + } +} + +string JoinPath(const Slice& a, const Slice& b) { + if (b.empty()) { + return a.ToString(); + } + if (!a.ends_with("/")) { + return Format("%s/%s", a, b); + } + return Format("%s%s", a, b); +} + +string HomeDir() { + return *kAppDir; +} + +string LibraryPath() { + return "Library"; +} + +string LibraryDir() { + return JoinPath(*kAppDir, LibraryPath()); +} + +string LoggingDir() { + return JoinPath(LibraryDir(), "Logs"); +} + +string LoggingQueueDir() { + return JoinPath(LoggingDir(), "Queue"); +} + +string TmpDir() { + return JoinPath(*kAppDir, TmpPath()); +} diff --git a/clients/shared/PathUtils.h b/clients/shared/PathUtils.h new file mode 100644 index 0000000..059a325 --- /dev/null +++ b/clients/shared/PathUtils.h @@ -0,0 +1,23 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_PATH_UTILS_H +#define VIEWFINDER_PATH_UTILS_H + +#import "Utils.h" + +#ifdef OS_ANDROID +void InitApplicationPath(const string& dir); +# else // !OS_ANDROID +string MainBundlePath(const string &filename); +#endif // !OS_ANDROID + +string JoinPath(const Slice& a, const Slice& b); +string HomeDir(); +string LibraryPath(); +string LibraryDir(); +string LoggingDir(); +string LoggingQueueDir(); +string TmpDir(); + +#endif // VIEWFINDER_PATH_UTILS_H diff --git a/clients/shared/PathUtils.ios.mm b/clients/shared/PathUtils.ios.mm new file mode 100644 index 0000000..230753c --- /dev/null +++ b/clients/shared/PathUtils.ios.mm @@ -0,0 +1,130 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "Defines.h" +#import "FileUtils.h" +#import "Format.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "PathUtils.h" +#import "StringUtils.h" + +namespace { + +string StripHomeDirPrefix(const Slice& s) { + Slice h([NSHomeDirectory() UTF8String]); + CHECK(s.starts_with(h)); + return s.substr(h.size() + 1).ToString(); +} + +string GetDirPath(NSSearchPathDirectory dir) { + NSArray* dirs = NSSearchPathForDirectoriesInDomains( + dir, NSUserDomainMask, YES); + if (dirs.count == 0) { + return string(); + } + return StripHomeDirPrefix([[dirs objectAtIndex:0] UTF8String]); +} + +string TmpPath() { + return "tmp"; +} + +class HomeDir : public string { + public: + HomeDir() { + string dir = ToString(NSHomeDirectory()); + const string clean_slate_dir = JoinPath(JoinPath(dir, TmpPath()), "CleanSlate"); + +#ifdef CLEAN_SLATE_VERSION + dir = clean_slate_dir; + DirCreate(dir); + + vector filenames; + DirList(dir, &filenames); + +#ifdef CLEAN_SLATE_RESTORE_VERSION + const string save_subdir = ToString(CLEAN_SLATE_RESTORE_VERSION); +#else // !CLEAN_SLATE_RESTORE_VERSION + const string save_subdir = ToString(CLEAN_SLATE_VERSION); +#endif // !CLEAN_SLATE_RESTORE_VERSION + + // List out all of the existing files/directories in "CleanSlate". Remove any + // that do not match the current version. + for (int i = 0; i < filenames.size(); ++i) { + if (filenames[i] != save_subdir) { + DirRemove(JoinPath(dir, filenames[i]), true); + } + } + + const string dest_dir = JoinPath(dir, ToString(CLEAN_SLATE_VERSION)); + +#ifdef CLEAN_SLATE_RESTORE_VERSION + { + const string restore_dir = JoinPath(dir, ToString(CLEAN_SLATE_RESTORE_VERSION)); + NSFileManager* fm = [NSFileManager defaultManager]; + NSError* error; + if (![fm copyItemAtPath:NewNSString(restore_dir) + toPath:NewNSString(dest_dir) + error:&error]) { + DIE("unable to copy %s to %s: %s", restore_dir, dest_dir, error); + } + } +#endif // !CLEAN_SLATE_RESTORE_VERSION + + dir = dest_dir; + DirCreate(dir); + DirCreate(JoinPath(dir, LibraryPath())); + DirCreate(JoinPath(dir, TmpPath())); +#else // !CLEAN_SLATE_VERSION + DirRemove(clean_slate_dir, true); +#endif // !CLEAN_SLATE_VERSION + + assign(dir); + } +}; + +LazyStaticPtr kHomeDir; + +} // namespace + +string MainBundlePath(const string &filename) { + NSString* name = [NSString stringWithUTF8String:filename.c_str()]; + NSString* path = [[NSBundle mainBundle] pathForResource:name ofType:nil]; + return ToString(path); +} + +string JoinPath(const Slice& a, const Slice& b) { + if (b.empty()) { + return a.ToString(); + } + if (!a.ends_with("/")) { + return Format("%s/%s", a, b); + } + return Format("%s%s", a, b); +} + +string HomeDir() { + return *kHomeDir; +} + +string LibraryPath() { + return GetDirPath(NSLibraryDirectory); +} + +string LibraryDir() { + return JoinPath(*kHomeDir, LibraryPath()); +} + +string LoggingDir() { + return JoinPath(LibraryDir(), "Logs"); +} + +string LoggingQueueDir() { + return JoinPath(LoggingDir(), "Queue"); +} + +string TmpDir() { + return JoinPath(*kHomeDir, TmpPath()); +} diff --git a/clients/shared/PeopleRank.cc b/clients/shared/PeopleRank.cc new file mode 100644 index 0000000..3ecb785 --- /dev/null +++ b/clients/shared/PeopleRank.cc @@ -0,0 +1,420 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball + +#import +#import "ActivityTable.h" +#import "AppState.h" +#import "PeopleRank.h" +#import "STLUtils.h" +#import "StringUtils.h" +#import "Timer.h" + +namespace { + +const string kFollowerGroupKeyPrefix = DBFormat::follower_group_key(""); + +const double kHalfLife = 45 * 24 * 60 * 60; // 1.5 months + +// This constant was empirically determined based on trying to +// minimize an annoying user who starts sometimes multiple +// conversations a day with me despite my utter lack of interest. But +// balanced to include groups from conversations I didn't start but +// still might. +const double kUserInitiatedConversationMultiplier = 4; + +// TODO(spencer): remove this in next version. +const DBRegisterKeyIntrospect kFollowerGroupKeyDeprecatedIntrospect( + DBFormat::follower_group_key_deprecated(""), NULL, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kFollowerGroupKeyIntrospect( + kFollowerGroupKeyPrefix, + [](Slice key) { + vector user_ids; + if (!DecodeFollowerGroupKey(key, &user_ids)) { + return string(); + } + return ToString(user_ids); + }, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +double Decay(double time) { + return exp(-log(2.0) * time / kHalfLife); +} + +struct ViewpointInfoLessThan { + bool operator()(const FollowerGroup::ViewpointInfo* a, + const FollowerGroup::ViewpointInfo* b) { + return a->viewpoint_id() < b->viewpoint_id(); + } + bool operator()(const FollowerGroup::ViewpointInfo* a, + const FollowerGroup::ViewpointInfo& b) { + return (*this)(a, &b); + } + bool operator()(const FollowerGroup::ViewpointInfo& a, + const FollowerGroup::ViewpointInfo& b) { + return (*this)(&a, &b); + } +}; + +struct BestContactGreaterThan { + PeopleRank* people_rank; + std::unordered_map* user_id_counts; + WallTime now; + + BestContactGreaterThan(PeopleRank* pr, + std::unordered_map* uic, + WallTime n) + : people_rank(pr), + user_id_counts(uic), + now(n) { + } + bool operator()(const ContactMetadata& a, const ContactMetadata& b) { + DCHECK(a.has_user_id()); + DCHECK(b.has_user_id()); + if ((*user_id_counts)[a.user_id()] != (*user_id_counts)[b.user_id()]) { + return (*user_id_counts)[a.user_id()] > (*user_id_counts)[b.user_id()]; + } + return people_rank->UserRank(a.user_id(), now) > people_rank->UserRank(b.user_id(), now); + } +}; + +struct FollowerGroupGreaterThan { + WallTime now; + FollowerGroupGreaterThan(WallTime n) + : now(n) { + } + bool operator()(const FollowerGroup* a, const FollowerGroup* b) { + return a->weight() > b->weight(); + } +}; + +struct ViewpointLatestTimestampGreaterThan { + bool operator()(const FollowerGroup::ViewpointInfo*& a, + const FollowerGroup::ViewpointInfo*& b) { + return a->latest_timestamp() > b->latest_timestamp(); + } +}; + +} // namespace + +typedef google::protobuf::RepeatedPtrField ViewpointInfoArray; + +string EncodeFollowerGroupKey(const vector& user_ids) { + vector sorted_user_ids(user_ids); + std::sort(sorted_user_ids.begin(), sorted_user_ids.end()); + string group_key; + for (int i = 0; i < sorted_user_ids.size(); ++i) { + OrderedCodeEncodeVarint64(&group_key, sorted_user_ids[i]); + } + return DBFormat::follower_group_key(group_key); +} + +bool DecodeFollowerGroupKey(Slice key, vector* user_ids) { + user_ids->clear(); + if (!key.starts_with(kFollowerGroupKeyPrefix)) { + return false; + } + key.remove_prefix(kFollowerGroupKeyPrefix.size()); + while (!key.empty()) { + user_ids->push_back(OrderedCodeDecodeVarint64(&key)); + } + return true; +} + +//// +// PeopleRank + +PeopleRank::PeopleRank(AppState* state) + : state_(state), + day_table_epoch_(0) { + Initialize(); +} + +PeopleRank::~PeopleRank() { +} + +double PeopleRank::UserRank(int64_t user_id, WallTime now) { + if (!ContainsKey(user_map_, user_id)) { + return 0; + } + return user_map_[user_id]; +} + +typedef google::protobuf::RepeatedField UserIdArray; + +// Note this is a brute force scan over all of the groups. It's fast +// enough for now, with a test case of 700 groups. However, we may +// need to keep an index from user id to group. Then you intersect +// the various groups, similar to the way ContactManager::Search() +// intersects the contacts matching the search. +void PeopleRank::FindBestGroups( + const vector& user_ids, vector* groups) { + CHECK(dispatch_is_main_thread()); + groups->clear(); + + for (GroupMap::iterator iter = group_map_.begin(); + iter != group_map_.end(); + ++iter) { + const FollowerGroup& fg = iter->second; + // Prospective groups must have more user ids than were passed. + if (fg.user_ids_size() <= user_ids.size()) { + continue; + } + bool all_match = true; + for (int i = 0; i < user_ids.size(); ++i) { + UserIdArray::const_iterator it = std::lower_bound( + fg.user_ids().begin(), fg.user_ids().end(), user_ids[i]); + if (it == fg.user_ids().end() || *it != user_ids[i]) { + all_match = false; + break; + } + } + + if (all_match) { + groups->push_back(&fg); + } + } + + std::sort(groups->begin(), groups->end(), + FollowerGroupGreaterThan(state_->WallTime_Now())); +} + +void PeopleRank::FindBestContacts( + const vector& user_ids, ContactManager::ContactVec* contacts) { + CHECK(dispatch_is_main_thread()); + vector groups; + FindBestGroups(user_ids, &groups); + + std::unordered_map user_id_counts; + for (int i = 0; i < user_ids.size(); ++i) { + user_id_counts[user_ids[i]] = 0; + } + + contacts->clear(); + for (int i = 0; i < groups.size(); ++i) { + for (int j = 0; j < groups[i]->user_ids_size(); ++j) { + const int64_t user_id = groups[i]->user_ids(j); + if (!ContainsKey(user_id_counts, user_id)) { + contacts->resize(contacts->size() + 1); + state_->contact_manager()->LookupUser(user_id, &contacts->back()); + } + ++user_id_counts[user_id]; + } + } + + std::sort(contacts->begin(), contacts->end(), BestContactGreaterThan(this, &user_id_counts, state_->WallTime_Now())); +} + +void PeopleRank::AddViewpoint( + int64_t viewpoint_id, const vector& user_ids, const DBHandle& updates) { + const string key = EncodeFollowerGroupKey(user_ids); + FollowerGroup* fg = NULL; + if (!ContainsKey(group_map_, key)) { + fg = &group_map_[key]; + for (int i = 0; i < user_ids.size(); ++i) { + fg->add_user_ids(user_ids[i]); + } + } else { + fg = &group_map_[key]; + } + + FollowerGroup::ViewpointInfo* vp_info = NULL; + FollowerGroup::ViewpointInfo search; + search.set_viewpoint_id(viewpoint_id); + ViewpointInfoArray::pointer_iterator it = + std::lower_bound(fg->mutable_viewpoints()->pointer_begin(), + fg->mutable_viewpoints()->pointer_end(), + &search, ViewpointInfoLessThan()); + if (it == fg->mutable_viewpoints()->pointer_end() || + (*it)->viewpoint_id() != viewpoint_id) { + // Viewpoint doesn't exist yet in this follower group; add it. + vp_info = fg->add_viewpoints(); + vp_info->set_viewpoint_id(viewpoint_id); + // Need to lookup first activity for earliest timestamp. + ActivityHandle ah = state_->activity_table()->GetFirstActivity(viewpoint_id, updates); + if (ah.get()) { + vp_info->set_earliest_timestamp(ah->timestamp()); + } + // Sort the newly added viewpoint. + std::sort(fg->mutable_viewpoints()->pointer_begin(), + fg->mutable_viewpoints()->pointer_end(), + ViewpointInfoLessThan()); + } + updates->PutProto(key, *fg); +} + +void PeopleRank::RemoveViewpoint( + int64_t viewpoint_id, const vector& user_ids, const DBHandle& updates) { + const string key = EncodeFollowerGroupKey(user_ids); + if (!ContainsKey(group_map_, key)) { + return; + } + FollowerGroup* fg = &group_map_[key]; + + FollowerGroup::ViewpointInfo search; + search.set_viewpoint_id(viewpoint_id); + ViewpointInfoArray::pointer_iterator it = + std::lower_bound(fg->mutable_viewpoints()->pointer_begin(), + fg->mutable_viewpoints()->pointer_end(), + &search, ViewpointInfoLessThan()); + if (it == fg->mutable_viewpoints()->pointer_end() || + (*it)->viewpoint_id() != viewpoint_id) { + // Viewpoint isn't here; ignore. + return; + } + if (fg->viewpoints_size() == 0) { + updates->Delete(key); + group_map_.erase(key); + } else { + updates->PutProto(key, *fg); + } +} + +void PeopleRank::Reset() { + user_map_.clear(); + group_map_.clear(); +} + +vector PeopleRank::MostRecentViewpoints( + const FollowerGroup& group, int max_count) { + ViewpointLatestTimestampGreaterThan compare; + std::priority_queue, + ViewpointLatestTimestampGreaterThan> pq(compare); + for (int i = 0; i < group.viewpoints_size(); ++i) { + pq.push(&group.viewpoints(i)); + if (pq.size() > max_count) { + pq.pop(); + } + } + + vector vp_ids; + while (!pq.empty()) { + vp_ids.push_back(pq.top()->viewpoint_id()); + pq.pop(); + } + std::reverse(vp_ids.begin(), vp_ids.end()); + + return vp_ids; +} + +void PeopleRank::Initialize() { + for (DB::PrefixIterator iter(state_->db(), kFollowerGroupKeyPrefix); + iter.Valid(); + iter.Next()) { + FollowerGroup* fg = &group_map_[iter.key().as_string()]; + if (!fg->ParseFromArray(iter.value().data(), iter.value().size())) { + group_map_.erase(iter.key().as_string()); + } + } + + for (GroupMap::iterator iter = group_map_.begin(); + iter != group_map_.end(); + ++iter) { + FollowerGroup& fg = iter->second; + double fg_weight = 0; + for (int i = 0; i < fg.viewpoints_size(); ++i) { + fg_weight += fg.viewpoints(i).weight(); + } + fg.set_weight(fg_weight); + + // Iterate over each user in the group and update the user weight. + for (int i = 0; i < fg.user_ids_size(); ++i) { + const int64_t user_id = fg.user_ids(i); + user_map_[user_id] += fg_weight; + } + } + + // Setup day table refresh callback. + state_->day_table()->update()->Add([this] { + DayTableRefresh(); + }); +} + +void PeopleRank::DayTableRefresh() { + CHECK(dispatch_is_main_thread()); + + // If currently refreshing, skip the update. We wait for the day + // table to quiesce. Otherwise, since refreshes are done from most + // recent to least recent, the last update timestamp would skip + // refreshes to old viewpoints. + if (state_->day_table()->refreshing()) { + return; + } + + const int old_epoch = day_table_epoch_; + snapshot_ = state_->day_table()->GetSnapshot(&day_table_epoch_); + if (old_epoch == day_table_epoch_) { + // Nothing to do. + return; + } + + WallTime now = state_->WallTime_Now(); + WallTimer timer; + + // Initialize a map from viewpoint id to viewpoint info with weight + // and latest timestamp. + std::unordered_map weight_map; + for (int i = 0; i < snapshot_->conversations()->row_count(); ++i) { + SummaryRow row; + if (snapshot_->conversations()->GetSummaryRow(i, &row)) { + FollowerGroup::ViewpointInfo& vp_info = weight_map[row.identifier()]; + vp_info.set_weight(row.weight()); + vp_info.set_latest_timestamp(row.timestamp()); + } + } + + // Compute new weights for follower groups and users. + user_map_.clear(); + for (GroupMap::iterator iter = group_map_.begin(); + iter != group_map_.end(); + ++iter) { + FollowerGroup& fg = iter->second; + double fg_weight = 0; + + for (int i = 0; i < fg.viewpoints_size(); ++i) { + const FollowerGroup::ViewpointInfo& vp_info = weight_map[fg.viewpoints(i).viewpoint_id()]; + const double vp_weight = vp_info.weight() * Decay(now - vp_info.earliest_timestamp()); + fg.mutable_viewpoints(i)->set_latest_timestamp(vp_info.latest_timestamp()); + fg.mutable_viewpoints(i)->set_weight(vp_weight); + fg_weight += vp_weight; + } + + fg.set_weight(fg_weight); + + // Iterate over each user in the group and update the user weight. + for (int i = 0; i < fg.user_ids_size(); ++i) { + const int64_t user_id = fg.user_ids(i); + user_map_[user_id] += fg_weight; + } + } + + VLOG("people rank: refreshed in %.3fms", timer.Milliseconds()); + +#ifdef DEBUG_PEOPLE_RANK + for (GroupMap::iterator iter = group_map_.begin(); + iter != group_map_.end(); + ++iter) { + vector user_ids; + if (!DecodeFollowerGroupKey(iter->first, &user_ids)) { + LOG("group %s: %s", iter->first, iter->second); + } else { + LOG("group %s: %s", ToString(user_ids), iter->second); + } + } + for (UserMap::iterator iter = user_map_.begin(); + iter != user_map_.end(); + ++iter) { + LOG("user %d: %f", iter->first, iter->second); + } +#endif // DEBUG_PEOPLE_RANK +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PeopleRank.h b/clients/shared/PeopleRank.h new file mode 100644 index 0000000..db46da8 --- /dev/null +++ b/clients/shared/PeopleRank.h @@ -0,0 +1,120 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball + +#ifndef VIEWFINDER_PEOPLE_RANK_H +#define VIEWFINDER_PEOPLE_RANK_H + +#import +#import "ContactManager.h" +#import "DayTable.h" +#import "FollowerGroup.pb.h" +#import "Mutex.h" + +// Algorithm to compute weight for a group and for all users who belong +// to one or more groups. +// +// G: Follower group +// U: User +// UM: User weight map +// V: Viewpoint +// VW: viewpoint weight +// VT: time between "now" and creation timestamp of conversation +// DW: decayed viewpoint weight +// GW: group weight +// HL: half life of conversation weight +// +// for G in FollowerGroup.iteration: +// GW = 0; +// for V in G.viewpoints: +// DW = exp(-log(2.0) * VT / HL) * VW; +// GW += DW; +// for U in G.users: +// UM[U] += DW; + +// The PeopleRank class maintains the mapping: +// -> + +class PeopleRank { + public: + PeopleRank(AppState* state); + ~PeopleRank(); + + // NOTE: the public interface to PeopleRank should be used from the main + // thread only--it is not thread safe with possible asynchronous updates + // from the day table. + + // Returns a weight to be used for relative ranking between individual + // users which takes time and participation in all relevant viewpoints + // into consideration. Returns 0 if the user_id is unknown or the + // rankings are being determined. + // When using UserRank in a sorting function, the same value for 'now' + // must be used for all comparisons in the sort. + double UserRank(int64_t user_id, WallTime now); + + // Returns a sorted list of FollowerGroups, ordered by weight in + // descending order, which include all of the users listed in + // "user_ids", plus at least one additional user. + void FindBestGroups( + const vector& user_ids, vector* groups); + + // Returns a sorted list of ContactMetadata objects, ordered by weight in + // descending order. The list of applicable contacts is determined based on + // groups which also include the users listed in "user_ids". + void FindBestContacts( + const vector& user_ids, ContactManager::ContactVec* contacts); + + // Adds the viewpoint to the follower group containing the specified + // list of user ids. The vector of user ids need not be sorted. + void AddViewpoint( + int64_t viewpoint_id, const vector& user_ids, const DBHandle& updates); + + // Removes the viewpoint from the follower group containing the specified + // list of user ids. The vector of user ids need not be sorted. + void RemoveViewpoint( + int64_t viewpoint_id, const vector& user_ids, const DBHandle& updates); + + // Reset the internal state. Intended for use from migrations. + void Reset(); + + // Returns the "max_count" most recently updated viewpoints from the group. + static vector MostRecentViewpoints( + const FollowerGroup& group, int max_count); + + private: + enum UpdateType { + ADD_WEIGHT, + REMOVE_WEIGHT, + UPDATE_WEIGHT, + }; + + // Initialize the in-memory group and user weights. + void Initialize(); + + // Trolls the day table for updated viewpoints and adjusts last + // update times and weights as appropriate to recompute all group + // and user weights incrementally. + void DayTableRefresh(); + + void UpdateWeightsInternal( + int64_t viewpoint_id, const string& key, UpdateType u_type, const DBHandle& updates); + + private: + AppState* state_; + int day_table_epoch_; + DayTable::SnapshotHandle snapshot_; + // Comma-separated user ids => group weight. + typedef std::unordered_map GroupMap; + GroupMap group_map_; + // User id => user weight. + typedef std::unordered_map UserMap; + UserMap user_map_; +}; + +string EncodeFollowerGroupKey(const vector& user_ids); +bool DecodeFollowerGroupKey(Slice key, vector* user_ids); + +#endif // VIEWFINDER_PEOPLE_RANK_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PhoneUtils.cc b/clients/shared/PhoneUtils.cc new file mode 100644 index 0000000..bf1fc61 --- /dev/null +++ b/clients/shared/PhoneUtils.cc @@ -0,0 +1,60 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#import +#import +#import +#import +#import "LazyStaticPtr.h" +#import "PhoneUtils.h" +#import "ScopedPtr.h" + +using namespace i18n::phonenumbers; + +namespace { + +LazyStaticPtr kPhonePrefixRE = { "^[-+() 0-9]*$" }; + +} // namespace + +bool IsValidPhoneNumber(const string& s, const string& country_code) { + PhoneNumber number; + PhoneNumberUtil::ErrorType error = + PhoneNumberUtil::GetInstance()->Parse(s, country_code, &number); + if (error != PhoneNumberUtil::NO_PARSING_ERROR) { + return false; + } + return PhoneNumberUtil::GetInstance()->IsValidNumber(number); +} + +bool IsPhoneNumberPrefix(const string& s) { + return RE2::FullMatch(s, *kPhonePrefixRE); +} + +string FormatPhoneNumberPrefix(const string& s, const string& country_code) { + ScopedPtr formatter( + PhoneNumberUtil::GetInstance()->GetAsYouTypeFormatter(country_code)); + string result; + for (int i = 0; i < s.size(); i++) { + if (IsPhoneDigit(s[i])) { + formatter->InputDigit(s[i], &result); + } + } + return result; +} + +string NormalizedPhoneNumber(const string& s, const string& country_code) { + PhoneNumber number; + PhoneNumberUtil::ErrorType error = + PhoneNumberUtil::GetInstance()->Parse(s, country_code, &number); + if (error != PhoneNumberUtil::NO_PARSING_ERROR) { + return ""; + } + string result; + PhoneNumberUtil::GetInstance()->Format(number, PhoneNumberUtil::E164, &result); + return result; +} + +bool IsPhoneDigit(int chr) { + return u_isdigit(chr) || chr == '+'; +} diff --git a/clients/shared/PhoneUtils.h b/clients/shared/PhoneUtils.h new file mode 100644 index 0000000..28cf152 --- /dev/null +++ b/clients/shared/PhoneUtils.h @@ -0,0 +1,23 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#import "Utils.h" + +// Returns true if the given string is a valid phone number in the +// current locale. +bool IsValidPhoneNumber(const string& s, const string& country_code); + +// Returns true if the string could be the beginning of a phone number (i.e. it +// contains only digits and formatting characters). +bool IsPhoneNumberPrefix(const string& s); + +// Returns a partial number formatted for the given locale. See also +// PhoneNumberFormatter for additional integration with a UITextField. +string FormatPhoneNumberPrefix(const string& s, const string& country_code); + +// Returns a normalized (E164) version of the given phone number. +string NormalizedPhoneNumber(const string& s, const string& country_code); + +// Digits and plus signs are considered significant on input; all other formatting characters are +// ignored on input and inserted by the formatter. +bool IsPhoneDigit(int chr); diff --git a/clients/shared/PhotoMetadata.proto b/clients/shared/PhotoMetadata.proto new file mode 100644 index 0000000..ad46bae --- /dev/null +++ b/clients/shared/PhotoMetadata.proto @@ -0,0 +1,154 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +import "ContactMetadata.proto"; +import "ContentIds.proto"; +import "ImageFingerprint.proto"; +import "Location.proto"; +import "Placemark.proto"; +import "QueueMetadata.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "PhotoMetadataPB"; + +message PhotoMetadata { + /* + message Features { + message Point { + optional float x = 1; + optional float y = 2; + } + + message Rect { + optional float x = 1; + optional float y = 2; + optional float width = 3; + optional float height = 4; + } + + message Face { + // The bounds of the face in normalized image coordinates. All values are + // in the range [0,1). + optional Rect bounds = 1; + // The position of the left eye (if detected) in normalized image + // coordinates. + optional Point left_eye = 2; + // The position of the right eye (if detected) in normalized image + // coordinates. + optional Point right_eye = 3; + // The position of the mouth (if detected) in normalized image coordinates. + optional Point mouth = 4; + }; + + repeated Face faces = 1; + }; + */ + + message Image { + optional int32 size = 1; + optional string md5 = 2; + } + + message Images { + optional Image tn = 1; + optional Image med = 2; + optional Image full = 3; + optional Image orig = 4; + } + + enum PhotoType { + THUMBNAIL = 1; + MEDIUM = 2; + FULL = 4; + ORIGINAL = 8; + } + + optional PhotoId id = 1; + optional PhotoId parent_id = 2; + optional EpisodeId episode_id = 3; + optional int64 user_id = 4; + optional int64 sharing_user_id = 5; + optional int32 DEPRECATED_orientation = 6; + optional double aspect_ratio = 7; // width / height + optional double timestamp = 8; + optional Location location = 9; + optional Placemark placemark = 10; + optional string caption = 11; + optional string link = 12; + //optional Features features = 13; + optional Images images = 14; + optional QueueMetadata queue = 15; + // Has the photo been shared? Used to override a disabled cloud_storage and + // force the photo to be uploaded. + optional bool shared = 16; + optional string DEPRECATED_adjustment_xmp = 17; + // Full asset keys, i.e. a/url#fingerprint + repeated string asset_keys = 18; + // Bare fingerprints. Includes fingerprints of keys found in asset_keys. + // Append-only set. + repeated string asset_fingerprints = 19; + // The perceptual fingerprint. The terms inside of the fingerprint are an + // append-only set. + optional ImageFingerprint perceptual_fingerprint = 24; + // The list of candidate duplicate (local) photo ids for this photo. + repeated int64 candidate_duplicates = 25; + + // Bits indicating labels on the user-photo relation. + optional bool label_removed = 20; + optional bool label_hidden = 23; + optional bool label_unshared = 21; + // TODO(spencer): figure out what to do with this error label. I think + // it'll need to be reported to the server via an update_photo request. + optional bool label_error = 22; + + // Bits indicating network activity that is required for the photo. + optional bool update_metadata = 49; + optional bool upload_metadata = 40; + optional bool upload_thumbnail = 41; + optional bool upload_medium = 42; + optional bool upload_full = 43; + optional bool upload_original = 44; + optional bool download_thumbnail = 45; + optional bool download_medium = 46; + optional bool download_full = 47; + optional bool download_original = 48; + + // Various error states the photo can be in. + optional bool error_update_metadata = 77; + optional bool error_upload_metadata = 75; + optional bool error_upload_thumbnail = 60; + optional bool error_upload_medium = 61; + optional bool error_upload_full = 62; + optional bool error_upload_original = 63; + optional bool error_download_thumbnail = 64; + optional bool error_download_medium = 65; + optional bool error_download_full = 66; + optional bool error_download_original = 67; + optional bool error_asset_thumbnail = 68; + optional bool error_asset_full = 69; + optional bool error_asset_original = 70; + optional bool error_ui_thumbnail = 71; + optional bool error_ui_full = 72; + optional bool error_ui_original = 74; + optional bool error_timestamp = 73; + optional bool error_timestamp_invalid = 76; + optional bool error_placemark_invalid = 78; + + // Has this photo's placemark been processed into the placemark + // histogram? + optional bool placemark_histogram = 90; +} + +message PhotoPathMetadata { + // The server_id of the photo. Only set once the photo has been uploaded to + // the server. + optional string server_id = 1; + // The md5 of the file contents. + optional string md5 = 2; + // The last access time of the file in seconds since the epoch. + optional uint32 access_time = 3; + // The size of the file contents. + optional int64 size = 4; + // The size of the parent image which was used to generate this image. + optional int32 parent_size = 5; +} diff --git a/clients/shared/PhotoSelection.cc b/clients/shared/PhotoSelection.cc new file mode 100644 index 0000000..b32197b --- /dev/null +++ b/clients/shared/PhotoSelection.cc @@ -0,0 +1,17 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import +#import "PhotoSelection.h" + +ostream& operator<<(ostream& os, const PhotoSelection& ps) { + os << "(" << ps.photo_id << ", " << ps.episode_id + << ", [" << (WallTime_Now() - ps.timestamp) << "s ago])"; + return os; +} + +PhotoSelectionVec SelectionSetToVec(const PhotoSelectionSet& s) { + PhotoSelectionVec v(s.begin(), s.end());; + std::sort(v.begin(), v.end(), SelectionLessThan()); + return v; +} diff --git a/clients/shared/PhotoSelection.h b/clients/shared/PhotoSelection.h new file mode 100644 index 0000000..f3f7d75 --- /dev/null +++ b/clients/shared/PhotoSelection.h @@ -0,0 +1,59 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_PHOTO_SELECTION_H +#define VIEWFINDER_PHOTO_SELECTION_H + +#import +#import "WallTime.h" + +struct PhotoSelection { + int64_t photo_id; + int64_t episode_id; + WallTime timestamp; + PhotoSelection() + : photo_id(0), episode_id(0), timestamp(0) { + } + PhotoSelection(int64_t pi, int64_t ei, WallTime t = 0) + : photo_id(pi), + episode_id(ei), + timestamp(!t ? WallTime_Now() : t) { + } +}; + +ostream& operator<<(ostream& os, const PhotoSelection& ps); + +struct PhotoSelectionHash { + size_t operator()(const PhotoSelection& ps) const { + // TODO(spencer): we need a hashing module. + const size_t kPrime = 31; + size_t result = kPrime + int(ps.photo_id ^ (ps.photo_id >> 32)); + return result * kPrime + int(ps.episode_id ^ (ps.episode_id >> 32)); + } +}; + +struct PhotoSelectionEqualTo { + bool operator()(const PhotoSelection& a, const PhotoSelection& b) const { + return a.photo_id == b.photo_id && a.episode_id == b.episode_id; + } +}; + +struct SelectionLessThan { + bool operator()(const PhotoSelection& a, const PhotoSelection& b) const { + return a.timestamp < b.timestamp; + } +}; + +typedef std::unordered_set PhotoSelectionSet; +typedef vector PhotoSelectionVec; + +// Convert photo selection set to vector. +PhotoSelectionVec SelectionSetToVec(const PhotoSelectionSet& s); + +#endif // VIEWFINDER_PHOTO_SELECTION_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PhotoStorage.cc b/clients/shared/PhotoStorage.cc new file mode 100644 index 0000000..6b5fc24 --- /dev/null +++ b/clients/shared/PhotoStorage.cc @@ -0,0 +1,916 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +// TODO(pmattis): Run FixLocalUsage() periodically even if local_bytes isn't +// negative. + +#import +#import +#import +#import "Analytics.h" +#import "AsyncState.h" +#import "DigestUtils.h" +#import "FileUtils.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "PathUtils.h" +#import "PhotoMetadata.pb.h" +#import "PhotoStorage.h" +#import "PhotoTable.h" +#import "ServerUtils.h" +#import "STLUtils.h" +#import "StringUtils.h" +#import "Timer.h" + +namespace { + +typedef PhotoStorage::Setting Setting; +const vector kSettings = + L(Setting(100LL * 1024 * 1024, "100 MB", "1,000 photos"), + Setting(500LL * 1024 * 1024, "500 MB", "5,000 photos"), + Setting(1LL * 1024 * 1024 * 1024, "1 GB", "10,000 photos"), + Setting(5LL * 1024 * 1024 * 1024, "5 GB", "50,000 photos"), + Setting(10LL * 1024 * 1024 * 1024, "10 GB", "100,000 photos")); + +const string kFormatKey = DBFormat::metadata_key("photo_storage_format"); +const string kFormatValue = "1"; +const string kLastGarbageCollectionKey = DBFormat::metadata_key("last_garbage_collection"); +const string kLocalBytesLimitKey = DBFormat::metadata_key("local_storage"); +const string kLocalBytesKeyKey = DBFormat::metadata_key("local_bytes"); +const string kLocalFilesKey = DBFormat::metadata_key("local_files"); +const string kPhotoPathKeyPrefix = DBFormat::photo_path_key(""); +const string kPhotoPathAccessKeyPrefix = DBFormat::photo_path_access_key(""); +const string kRemoteUsageKey = DBFormat::metadata_key("remote_usage"); + +// Maintain a minimum headroom of 10 MB. +const int64_t kMinHeadroom = 10 * 1024 * 1024; +const WallTime kTimeGranularity = 60; +const WallTime kDay = 24 * 60 * 60; +const int kMD5Size = 32; + +LazyStaticPtr kViewfinderPhotoIdRE = { "([0-9]+)-.*" }; +LazyStaticPtr kViewfinderPhotoSizeRE = { ".*-([0-9]+).*" }; + +const DBRegisterKeyIntrospect kPhotoPathKeyIntrospect( + kPhotoPathKeyPrefix, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kPhotoPathAccessKeyIntrospect( + kPhotoPathAccessKeyPrefix, [](Slice key) { + if (!key.starts_with(kPhotoPathAccessKeyPrefix)) { + return string(); + } + key.remove_prefix(kPhotoPathAccessKeyPrefix.size()); + const uint32_t t = OrderedCodeDecodeVarint32Decreasing(&key); + return string(Format("%s/%s", DBIntrospect::timestamp(t), key)); + }, NULL); + +string AccessKey(const string& filename, uint32_t access_time) { + string s; + OrderedCodeEncodeVarint32Decreasing(&s, access_time); + s.append(filename); + return DBFormat::photo_path_access_key(s); +} + +int64_t DefaultBytesLimit() { + // Set the default amount of local storage to use based on the user's total + // disk space. + const int64_t kGB = 1LL * 1024 * 1024 * 1024; + const int64_t total_disk_space = TotalDiskSpace(); + if (total_disk_space <= 8 * kGB) { + return kSettings[0].value; + } + if (total_disk_space <= 32 * kGB) { + return kSettings[2].value; + } + return kSettings[3].value; +} + +} // namespace + +const int kThumbnailSize = 120; +const int kMediumSize = 480; +const int kFullSize = 960; +const int kOriginalSize = 1000000000; + +string PhotoSizeSuffix(int size) { + if (size == kOriginalSize) { + return "orig"; + } + return Format("%04d", size); +} + +string PhotoFilename(const string& server_id, const string& name) { + return Format("%s-%s.jpg", server_id, name); +} + +string PhotoFilename(int64_t local_id, const string& name) { + return Format("%d-%s.jpg", local_id, name); +} + +string PhotoFilename(const string& server_id, int size) { + return PhotoFilename(server_id, PhotoSizeSuffix(size)); +} + +string PhotoFilename(const PhotoId& photo_id, const string& name) { + return PhotoFilename(photo_id.local_id(), name); +} + +string PhotoFilename(int64_t local_id, int size) { + return PhotoFilename(local_id, PhotoSizeSuffix(size)); +} + +string PhotoFilename(const PhotoId& photo_id, int size) { + return PhotoFilename(photo_id, PhotoSizeSuffix(size)); +} + +string PhotoThumbnailFilename(const PhotoId& photo_id) { + return PhotoFilename(photo_id, kThumbnailSize); +} + +string PhotoThumbnailFilename(int64_t local_id) { + return PhotoFilename(local_id, kThumbnailSize); +} + +string PhotoMediumFilename(const PhotoId& photo_id) { + return PhotoFilename(photo_id, kMediumSize); +} + +string PhotoMediumFilename(int64_t local_id) { + return PhotoFilename(local_id, kMediumSize); +} + +string PhotoFullFilename(const PhotoId& photo_id) { + return PhotoFilename(photo_id, kFullSize); +} + +string PhotoFullFilename(int64_t local_id) { + return PhotoFilename(local_id, kFullSize); +} + +string PhotoOriginalFilename(const PhotoId& photo_id) { + return PhotoFilename(photo_id, kOriginalSize); +} + +string PhotoOriginalFilename(int64_t local_id) { + return PhotoFilename(local_id, kOriginalSize); +} + +string PhotoBasename(const string& dir, const string& path) { + Slice s(path); + s.remove_prefix(dir.size() + 1); + return s.ToString(); +} + +int64_t PhotoFilenameToLocalId(const Slice& filename) { + int64_t local_id = -1; + RE2::FullMatch(filename, *kViewfinderPhotoIdRE, &local_id); + return local_id; +} + +int PhotoFilenameToSize(const Slice& filename) { + int size = kOriginalSize; + RE2::FullMatch(filename, *kViewfinderPhotoSizeRE, &size); + return size; +} + +int PhotoFilenameToType(const Slice& filename) { + const int size = PhotoFilenameToSize(filename); + switch (size) { + case kThumbnailSize: + return FILE_THUMBNAIL; + case kMediumSize: + return FILE_MEDIUM; + case kFullSize: + return FILE_FULL; + case kOriginalSize: + default: + return FILE_ORIGINAL; + } +} + +PhotoStorage::Scanner::Scanner(PhotoStorage* photo_storage) + : photo_storage_(photo_storage), + iter_(db()->NewIterator()), + local_bytes_(0) { + memset(local_files_, 0, sizeof(local_files_)); + photo_storage_->AddScanner(this); +} + +PhotoStorage::Scanner::~Scanner() { + photo_storage_->RemoveScanner(this); +} + +bool PhotoStorage::Scanner::Step(int num_files) { + MutexLock l(&mu_); + for (int i = 0; num_files < 0 || i < num_files; ++i) { + // Only advance the iterator just before we will use its value. + if (pos_.empty()) { + iter_->Seek(kPhotoPathKeyPrefix); + } else if (iter_->Valid()) { + iter_->Next(); + } + if (!iter_->Valid()) { + return false; + } + const Slice key(ToSlice(iter_->key())); + if (!key.starts_with(kPhotoPathKeyPrefix)) { + // We've reached the end of the photo_path_key keys. Set pos_ to the + // "inifity" key so that the scanner will capture all future local_usage + // increments. + pos_ = "\xff"; + return false; + } + // Set pos_ to the name of the last file the scanner examined. + pos_ = key.substr(kPhotoPathKeyPrefix.size()); + const Slice value(ToSlice(iter_->value())); + PhotoPathMetadata m; + m.ParseFromArray(value.data(), value.size()); + if (m.has_size()) { + local_bytes_ += m.size(); + } else { + local_bytes_ += FileSize(photo_storage_->PhotoPath(pos_)); + } + local_files_[PhotoFilenameToType(pos_)] += 1; + } + return true; +} + +void PhotoStorage::Scanner::StepNAndBackground( + int files_per_step, Callback done) { + if (!Step(files_per_step)) { + done(); + return; + } + // Step through the remaining files on a background thread. + photo_storage_->async_->dispatch_background([this, done] { + Step(); + done(); + }); +} + +void PhotoStorage::Scanner::IncrementLocalUsage( + const string& filename, int64_t delta) { + MutexLock l(&mu_); + // Note that pos_ is the name of the last file the scanner examined. Only + // apply the delta if it is for a filename that has already been + // scanned. + // + // Note the use of <= is critical because we don't want there to be any + // window where a file can slip in. For example, if pos_ were instead the + // name of the next file the scanner will examine and the comparison below + // was "filename < pos_" it would be possible for a file to be added to the + // photo store but slip past the scanner. Consider the scanner saw the file + // "aa" and the next file it will examine is "bb" (i.e. pos="bb"). If the + // file "b" is created ("aa" < "b" < "bb"), the scanner would miss it. + if (filename <= pos_) { + local_bytes_ += delta; + local_files_[PhotoFilenameToType(filename)] += (delta < 0) ? -1 : +1; + } +} + +PhotoStorage::PhotoStorage(AppState* state) + : state_(state), + async_(new AsyncState(state->async())), + photo_dir_(state_->photo_dir()), + server_photo_dir_(state_->server_photo_dir()), + local_bytes_limit_(state_->db()->Get( + kLocalBytesLimitKey, DefaultBytesLimit())), + local_bytes_(state_->db()->Get(kLocalBytesKeyKey, -1)), + gc_(true) { + for (int i = 0; i < ARRAYSIZE(local_files_); ++i) { + local_files_[i] = state_->db()->Get( + Format("%s/%d", kLocalFilesKey, i), -1); + } + + if (local_bytes_ < 0 || + state_->db()->Get(kFormatKey) != kFormatValue) { + FixLocalUsage(); + } else { + VLOG("photo storage: local usage: %.2f MB", + local_bytes_ / (1024.0 * 1024.0)); + state_->analytics()->LocalUsage( + local_bytes_, local_files_[0], local_files_[1], + local_files_[2], local_files_[3]); + } + + if (state_->db()->GetProto(kRemoteUsageKey, &remote_usage_)) { + VLOG("remote usage: %s", remote_usage_); + } + + // Only kick off garbage collection if it has been more than a day since + // garbage collection was last run. + const WallTime last_garbage_collection = + state_->db()->Get(kLastGarbageCollectionKey); + if (WallTime_Now() - last_garbage_collection >= kDay) { + if (last_garbage_collection == 0) { + // No need to garbage collect the first time that app is started. + state_->db()->Put(kLastGarbageCollectionKey, WallTime_Now()); + } else { + async_->dispatch_after_background( + 15, [this, last_garbage_collection] { + LOG("photo storage: garbage collect: last collection: %s", + WallTimeFormat("%F %T", last_garbage_collection)); + GarbageCollect(); + }); + } + } +} + +PhotoStorage::~PhotoStorage() { + async_.reset(NULL); +} + +bool PhotoStorage::Write( + const string& filename, int parent_size, + const Slice& data, const DBHandle& updates) { + const string key = DBFormat::photo_path_key(filename); + if (IsUncommittedFile(filename) || updates->Exists(key)) { + // Don't overwrite an existing file. + return false; + } + + updates->AddCommitTrigger(key, [this, filename]{ + RemoveUncommittedFile(filename); + }); + AddUncommittedFile(filename); + + // TODO(pmattis): Gracefully handle out-of-space errors. + if (!WriteStringToFile(PhotoPath(filename), data)) { + RemoveUncommittedFile(filename); + return false; + } + // Update the last access time. + PhotoPathMetadata m; + m.set_md5(MD5(data)); + m.set_access_time(state_->WallTime_Now()); + m.set_size(data.size()); + if (parent_size > 0) { + m.set_parent_size(parent_size); + } + updates->PutProto(key, m); + updates->Put(AccessKey(filename, m.access_time()), string()); + IncrementLocalUsage(filename, m.size(), updates); + return true; +} + +bool PhotoStorage::AddExisting( + const string& path, const string& filename, + const string& md5, const string& server_id, const DBHandle& updates) { + if (IsUncommittedFile(filename)) { + // Don't overwrite an existing file. + return false; + } + + // Protect "filename" from being removed until the database update commits. + const string key = DBFormat::photo_path_key(filename); + updates->AddCommitTrigger(key, [this, filename] { + RemoveUncommittedFile(filename); + }); + AddUncommittedFile(filename); + + // TODO(pmattis): Gracefully handle out-of-space errors. + const string new_path = PhotoPath(filename); + if (!FileRename(path, new_path)) { + RemoveUncommittedFile(filename); + return false; + } + // Update the last access time. + PhotoPathMetadata m; + if (!server_id.empty()) { + const string server_path = PhotoServerPath(filename, server_id); + if (link(new_path.c_str(), server_path.c_str()) == 0) { + m.set_server_id(server_id); + } + } + m.set_md5(md5); + m.set_access_time(state_->WallTime_Now()); + m.set_size(FileSize(new_path)); + updates->PutProto(key, m); + updates->Put(AccessKey(filename, m.access_time()), string()); + IncrementLocalUsage(filename, m.size(), updates); + return true; +} + +void PhotoStorage::SetServerId( + const string& filename, const string& server_id, const DBHandle& updates) { + const string key = DBFormat::photo_path_key(filename); + PhotoPathMetadata m; + if (!state_->db()->GetProto(key, &m)) { + return; + } + + // Construct the old and new paths. + const string old_path = PhotoPath(filename); + const string new_path = PhotoServerPath(filename, server_id); + + // Update the metadata to point to the new path. + m.set_server_id(server_id); + state_->db()->PutProto(key, m); + + // Link the old path to the new path. + link(old_path.c_str(), new_path.c_str()); +} + +void PhotoStorage::SetAssetSymlink( + const string& filename, const string& server_id, + const string& asset_key) { + const string symlink_path = PhotoServerPath(filename, server_id); + FileRemove(symlink_path); + if (symlink(asset_key.c_str(), symlink_path.c_str()) == -1) { + LOG("photo storage: symlink failed: %s -> %s: %d (%s)", + asset_key, symlink_path, errno, strerror(errno)); + } +} + +string PhotoStorage::ReadAssetSymlink( + const string& filename, const string& server_id) { + const string symlink_path = PhotoServerPath(filename, server_id); + char buf[1024]; + int n = readlink(symlink_path.c_str(), buf, ARRAYSIZE(buf)); + if (n == -1) { + return string(); + } + return string(buf, n); +} + +bool PhotoStorage::HaveAssetSymlink( + const string& filename, const string& server_id, + const string& asset_key) { + const string asset_symlink = ReadAssetSymlink(filename, server_id); + if (asset_symlink.empty()) { + return false; + } + Slice url; + Slice fingerprint; + if (!DecodeAssetKey(asset_key, &url, &fingerprint)) { + return false; + } + if (asset_symlink == fingerprint) { + // Symlinks are supposed to contain asset fingerprints. + return true; + } + Slice symlink_url; + Slice symlink_fingerprint; + if (!DecodeAssetKey(asset_symlink, &symlink_url, &symlink_fingerprint)) { + return false; + } + // But they used to contain asset urls. + if (symlink_url != url) { + return false; + } + // Replace the existing asset url symlink with the asset fingerprint. + SetAssetSymlink(filename, server_id, fingerprint.ToString()); + DCHECK_EQ(fingerprint, ReadAssetSymlink(filename, server_id)); + return true; +} + +void PhotoStorage::Delete(const string& filename, const DBHandle& updates) { + PhotoPathMetadata m; + const bool exists = + state_->db()->GetProto(DBFormat::photo_path_key(filename), &m); + // Delete the image before deleting from the database. If we crash before + // deleting from the database, we'll just attempt another deletion the next + // time we startup. + const string path = PhotoPath(filename); + const int64_t size = m.has_size() ? m.size() : FileSize(path); + VLOG("photo storage: delete %s: %d", filename, size); + // Note that we remove the file on disk even if the metadata exists because + // we might be called via the garbage collection code path. + FileRemove(path); + if (m.has_server_id()) { + FileRemove(PhotoServerPath(filename, m.server_id())); + } + if (m.has_access_time()) { + updates->Delete(AccessKey(filename, m.access_time())); + } + if (exists) { + IncrementLocalUsage(filename, -size, updates); + } + updates->Delete(DBFormat::photo_path_key(filename)); +} + +void PhotoStorage::DeleteAll(int64_t photo_id, const DBHandle& updates) { + for (DB::PrefixIterator iter(updates, DBFormat::photo_path_key(Format("%s-", photo_id))); + iter.Valid(); + iter.Next()) { + Delete(iter.key().substr(kPhotoPathKeyPrefix.size()).ToString(), updates); + } +} + +string PhotoStorage::Read(const string& filename, const string& metadata) { + string s; + if (!ReadFileToString(PhotoPath(filename), &s)) { + return string(); + } + Touch(filename, metadata); + return s; +} + +void PhotoStorage::Touch(const string& filename, const string& metadata) { + // Decode the filename metadata. + PhotoPathMetadata m; + m.ParseFromString(metadata); + // Only update the access time if kTimeGranularity seconds have passed since + // the last update. + const WallTime now = state_->WallTime_Now(); + if (!m.has_access_time() || + (fabs(now - m.access_time()) >= kTimeGranularity)) { + // TODO(pmattis): Batch access time updates together so that they are only + // written once per X seconds. + DBHandle updates = state_->NewDBTransaction(); + updates->Delete(AccessKey(filename, m.access_time())); + m.set_access_time(now); + updates->PutProto(DBFormat::photo_path_key(filename), m); + updates->Put(AccessKey(filename, m.access_time()), string()); + updates->Commit(false); + } +} + +bool PhotoStorage::MaybeLinkServerId( + const string& filename, const string& server_id, + const string& md5, const DBHandle& updates) { + const string local_path = PhotoPath(filename); + if (updates->Exists(DBFormat::photo_path_key(filename)) && + FileExists(local_path)) { + return true; + } + + const string server_path = PhotoServerPath(filename, server_id); + const int64_t size = FileSize(server_path); + if (size <= 0) { + return false; + } + + // If the server didn't give us an MD5, don't trust the local data. + if (md5.empty()) { + FileRemove(server_path); + return false; + } + + if (link(server_path.c_str(), local_path.c_str()) != 0) { + return false; + } + + // Update the last access time. + PhotoPathMetadata m; + m.set_server_id(server_id); + m.set_md5(md5); + m.set_access_time(state_->WallTime_Now()); + m.set_size(size); + updates->PutProto(DBFormat::photo_path_key(filename), m); + updates->Put(AccessKey(filename, m.access_time()), string()); + IncrementLocalUsage(filename, m.size(), updates); + return true; +} + +bool PhotoStorage::Exists(const string& filename) { + return state_->db()->Exists(DBFormat::photo_path_key(filename)); +} + +int64_t PhotoStorage::Size(const string& filename) { + // Note that we want to check the actual file size on disk and not just the + // size in PhotoPathMetadata so that this doubles as a check for the file's + // existence. + return ::FileSize(PhotoPath(filename)); +} + +PhotoPathMetadata PhotoStorage::Metadata(const string& filename) { + PhotoPathMetadata m; + state_->db()->GetProto(DBFormat::photo_path_key(filename), &m); + return m; +} + +string PhotoStorage::LowerBound( + int64_t photo_id, int max_size, string* metadata) { + const string max_key = + DBFormat::photo_path_key(PhotoFilename(photo_id, max_size)); + string filename; + for (DB::PrefixIterator iter(state_->db(), DBFormat::photo_path_key(Format("%d-", photo_id))); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + filename = key.ToString(); + *metadata = value.ToString(); + if (key >= max_key) { + break; + } + } + if (filename.empty()) { + return string(); + } + // Strip off the prefix. + return filename.substr(kPhotoPathKeyPrefix.size()); +} + +void PhotoStorage::GarbageCollect() { + { + MutexLock l(&mu_); + if (!gc_) { + // We've already collected garbage in this instance of the app. Garbage + // is only created when the app crashes unexpectedly. + return; + } + gc_ = false; + } + + // PhotoStorage updates the filesystem before updating the database. See + // Write(), AddExisting() and Delete(). We want to garbage collect database + // keys for which there is not a corresponding file on disk. Before listing + // the on disk images, grab a snapshot of the database state. + DBHandle snapshot = state_->NewDBSnapshot(); + DBHandle updates = state_->NewDBTransaction(); + + StringSet files; + + { + // Populate a set of all of the filenames that exist in the photo + // directory. + LOG("photo storage: garbage collection: list files"); + vector files_vec; + DirList(photo_dir_, &files_vec); + for (int i = 0; i < files_vec.size(); ++i) { + // TODO(pmattis): This test for "tmp" can go away soon now that we're + // placing temporary (downloading) photos in /tmp/photos. + if (files_vec[i] != "tmp") { + files.insert(files_vec[i]); + } + } + } + + // Loop over all of the photo paths listed in the database and check to see + // that there is corresponding photo metadata. Garbage collect any photo path + // and the on disk image for which there is not associated photo metadata or + // for which the on-disk image does not exist. + LOG("photo storage: garbage collection: list database keys"); + for (DB::PrefixIterator iter(snapshot, kPhotoPathKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const string filename = key.substr(kPhotoPathKeyPrefix.size()).ToString(); + if (IsUncommittedFile(filename)) { + LOG("photo storage: garbage collecting (skipping uncommited file): %s", + filename); + continue; + } + const int64_t local_id = PhotoFilenameToLocalId(filename); + if (local_id == -1) { + continue; + } + if (!state_->photo_table()->Exists(local_id, snapshot)) { + // We have a photo-path-key for a photo that does not exist in the + // PhotoTable. + LOG("photo storage: garbage collecting (photo-id does not exist): %d: %s", + local_id, filename); + Delete(filename, updates); + } else if (!ContainsKey(files, filename) && + state_->db()->Exists(key) && + !FileExists(PhotoPath(filename))) { + // We have a photo-path-key for an image that does not exist on + // disk. It is possible a concurrent deletion of the image occurred, + // but it is safe for us to re-delete the image in that case. + // + // NOTE(peter): We need the !FileExists() test because the photo + // could have been deleted and recreated between the creation of the + // database snapshot and the listing of the image files on disk. The + // ContainsKey() test can be viewed as an optimization that avoids + // the FileExists() test in most cases. + LOG("photo storage: garbage collecting (image does not exist): %d: %s", + local_id, filename); + Delete(filename, updates); + } + } + + // Loop over the on disk images in the photo directory and garbage collect + // any image for which there is not a corresponding photo path in the + // database. + for (StringSet::iterator iter(files.begin()); + iter != files.end(); + ++iter) { + const string& filename = *iter; + if (IsUncommittedFile(filename)) { + LOG("photo storage: garbage collecting (skipping uncommited file): %s", + filename); + continue; + } + // The FileExists() check prevents spurious log messages in the case where + // the image is deleted between the time it is listed and now. + if (!state_->db()->Exists(DBFormat::photo_path_key(filename)) && + FileExists(PhotoPath(filename))) { + LOG("photo storage: garbage collecting (photo-path-key does not exist): %s", + filename); + Delete(filename, updates); + } + } + + updates->Put(kLastGarbageCollectionKey, WallTime_Now()); + updates->Commit(); +} + +bool PhotoStorage::Check() { + // Each filename listed under photo_path_key with a non-zero access time + // should have a corresponding entry under photo_path_access_key. + bool ok = true; + for (DB::PrefixIterator iter(state_->db(), kPhotoPathKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + PhotoPathMetadata m; + m.ParseFromArray(value.data(), value.size()); + if (m.has_access_time()) { + const string filename = + key.substr(kPhotoPathKeyPrefix.size()).ToString(); + if (!state_->db()->Exists(AccessKey(filename, m.access_time()))) { + LOG("photo storage: photo path access key does not exist: %s %s", + filename, WallTimeFormat("%F-%T", m.access_time())); + ok = false; + } + } + } + + // Each filename should be listed only once under photo_path_access_key. And + // every entry under photo_path_access_key should be pointed to by the + // PhotoPathMetadata under photo_path_key. + std::unordered_set filenames; + for (DB::PrefixIterator iter(state_->db(), kPhotoPathAccessKeyPrefix); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + Slice s(key.substr(kPhotoPathAccessKeyPrefix.size())); + const uint32_t access_time = OrderedCodeDecodeVarint32Decreasing(&s); + const string filename = s.ToString(); + if (ContainsKey(filenames, filename)) { + LOG("photo storage: filename occurs twice in photo path access: %s", + filename); + ok = false; + } + filenames.insert(filename); + + const string d = state_->db()->Get( + DBFormat::photo_path_key(filename), string()); + PhotoPathMetadata m; + m.ParseFromString(d); + if (key != AccessKey(filename, m.access_time())) { + LOG("photo storage: photo path/access time mismatch: %s != %s", + WallTimeFormat("%F-%T", access_time), + WallTimeFormat("%F-%T", m.access_time())); + ok = false; + } + } + + return ok; +} + +string PhotoStorage::PhotoPath(const Slice& filename) { + return JoinPath(photo_dir_, filename); +} + +string PhotoStorage::PhotoServerPath( + const Slice& filename, const string& server_id) { + if (server_id.empty()) { + return string(); + } + const string server_filename = PhotoFilename( + server_id, PhotoFilenameToSize(filename)); + return JoinPath(server_photo_dir_, server_filename); +} + +void PhotoStorage::IncrementLocalUsage( + const string& filename, int64_t delta, const DBHandle& updates) { + { + // The PhotoStorage class tracks the disk space used by all of the files + // under its control in the local_usage_ member. Under various crash + // scenarious, this value can get out of date. The Scanner class exists to + // refresh the local_usage_ value. A scanner works by iterating over the + // filenames in a photo storage in sorted order. The tricky part is that we + // don't pause the photo storage while the scanner is operating. The user + // might add a new picture or delete an existing picture. We keep the + // scanner computation correct by telling any active scanners about changes + // to files and letting the scanner determine if it should apply the change + // or not based on whether it has already examined the file or not. + MutexLock l(&mu_); + for (ScannerSet::iterator iter(scanners_.begin()); + iter != scanners_.end(); + ++iter) { + (*iter)->IncrementLocalUsage(filename, delta); + } + + // If the local_usage value is invalid, don't adjust it further. + if (local_bytes_ < 0) { + return; + } + + const int type = PhotoFilenameToType(filename); + local_bytes_ += delta; + local_files_[type] += (delta < 0) ? -1 : +1; + if (local_bytes_ >= 0) { + updates->Put(kLocalBytesKeyKey, local_bytes_); + updates->Put(string(Format("%s/%d", kLocalFilesKey, type)), + local_files_[type]); + // Run the changed callback after we've released mu_. + async_->dispatch_after_main(0, [this] { + changed_.Run(); + }); + return; + } + } + + // This increment whacked the local_usage value out of alignment. Fire off a + // scanner to fix it. + FixLocalUsage(); +} + +void PhotoStorage::FixLocalUsage() { + WallTimer timer; + Scanner* scanner = new Scanner(this); + scanner->StepNAndBackground(100, [this, scanner, timer] { + ScopedPtr scanner_deleter(scanner); + + MutexLock l(&mu_); + local_bytes_ = scanner->local_bytes(); + for (int i = 0; i < ARRAYSIZE(local_files_); ++i) { + local_files_[i] = scanner->local_files(i); + } + LOG("photo storage: fixed local usage: bytes=%d, files=%d: %.3f ms", + local_bytes_, local_files_[0] + local_files_[1] + + local_files_[2] + local_files_[3], + timer.Milliseconds()); + DBHandle updates = state_->NewDBTransaction(); + updates->Put(kLocalBytesKeyKey, local_bytes_); + for (int i = 0; i < ARRAYSIZE(local_files_); ++i) { + updates->Put(string(Format("%s/%d", kLocalFilesKey, i)), + local_files_[i]); + } + updates->Put(kFormatKey, kFormatValue); + updates->Commit(); + state_->analytics()->LocalUsage( + local_bytes_, local_files_[0], local_files_[1], + local_files_[2], local_files_[3]); + // Run the changed callback on the main thread and after we've released + // mu_. + async_->dispatch_after_main(0, [this] { + changed_.Run(); + }); + }); +} + +void PhotoStorage::AddScanner(Scanner* scanner) { + MutexLock l(&mu_); + scanners_.insert(scanner); +} + +void PhotoStorage::RemoveScanner(Scanner* scanner) { + MutexLock l(&mu_); + scanners_.erase(scanner); +} + +void PhotoStorage::AddUncommittedFile(const string& filename) { + MutexLock l(&mu_); + uncommitted_files_.insert(filename); +} + +void PhotoStorage::RemoveUncommittedFile(const string& filename) { + MutexLock l(&mu_); + uncommitted_files_.erase(filename); +} + +bool PhotoStorage::IsUncommittedFile(const string& filename) { + MutexLock l(&mu_); + return ContainsKey(uncommitted_files_, filename); +} + +void PhotoStorage::set_local_bytes_limit(int64_t v) { + local_bytes_limit_ = v; + state_->db()->Put(kLocalBytesLimitKey, local_bytes_limit_); +} + +const vector& PhotoStorage::settings() const { + return kSettings; +} + +int PhotoStorage::setting_index(int64_t value) const { + for (int i = 0; i < kSettings.size(); ++i) { + if (value <= kSettings[i].value) { + return i; + } + } + return kSettings.size() - 1; +} + +void PhotoStorage::update_remote_usage(const UsageMetadata& usage) { + MutexLock l(&mu_); + remote_usage_.MergeFrom(usage); + state_->db()->PutProto(kRemoteUsageKey, remote_usage_); +} + +UsageMetadata PhotoStorage::remote_usage() const { + MutexLock l(&mu_); + return remote_usage_; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PhotoStorage.h b/clients/shared/PhotoStorage.h new file mode 100644 index 0000000..97cb594 --- /dev/null +++ b/clients/shared/PhotoStorage.h @@ -0,0 +1,220 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_PHOTO_STORAGE_H +#define VIEWFINDER_PHOTO_STORAGE_H + +#import +#import "AppState.h" +#import "Callback.h" +#import "DB.h" +#import "Mutex.h" +#import "UserMetadata.pb.h" + +class AsyncState; +class PhotoPathMetadata; + +extern const int kThumbnailSize; +extern const int kMediumSize; +extern const int kFullSize; +extern const int kOriginalSize; + +enum PhotoFileType { + FILE_THUMBNAIL = 0, + FILE_MEDIUM = 1, + FILE_FULL = 2, + FILE_ORIGINAL = 3, +}; + +// Various routines for generate and manipulating photo filenames. +string PhotoSizeSuffix(int size); +string PhotoFilename(const string& server_id, const string& name); +string PhotoFilename(const string& server_id, int size); +string PhotoFilename(int64_t local_id, const string& name); +string PhotoFilename(const PhotoId& photo_id, const string& name); +string PhotoFilename(int64_t local_id, int size); +string PhotoFilename(const PhotoId& photo_id, int size); +string PhotoThumbnailFilename(const PhotoId& photo_id); +string PhotoThumbnailFilename(int64_t local_id); +string PhotoMediumFilename(const PhotoId& photo_id); +string PhotoMediumFilename(int64_t local_id); +string PhotoFullFilename(const PhotoId& photo_id); +string PhotoFullFilename(int64_t local_id); +string PhotoOriginalFilename(const PhotoId& photo_id); +string PhotoOriginalFilename(int64_t local_id); +// Extracts the base photo filename from a full path. +string PhotoBasename(const string& dir, const string& path); +// Extracts the local photo id from a filename. +int64_t PhotoFilenameToLocalId(const Slice& filename); +// Extracts the size from a filename. +int PhotoFilenameToSize(const Slice& filename); +int PhotoFilenameToType(const Slice& filename); + +// The photo storage class is the low-level interface for writing, reading and +// deleting photos on disk. In addition to wrapping the filesystem routines, it +// tracks access times and how much disk space is being used and reclaims disk +// space when a configurable threshold is reached. +class PhotoStorage { + public: + // A scanner allows asynchronous iteration over the files in a photo storage, + // computing a value (currently just local-usage) as it goes. Care is taken + // to correctly handle mutations to the photo store that occur concurrently + // with the scan. + class Scanner { + public: + Scanner(PhotoStorage* photo_storage); + ~Scanner(); + + // Step through the next num_files files. + bool Step(int num_files = -1); + // Asynchronously step through all of the remaining files in the + // scanner. The first step is performed synchronously in + // files_per_step. The done block is called when the scanner reaches the + // end. + void StepNAndBackground(int files_per_step, Callback done); + void IncrementLocalUsage(const string& filename, int64_t delta); + + const DBHandle& db() { return photo_storage_->state_->db(); } + const Slice& pos() const { return pos_; } + int64_t local_bytes() const { return local_bytes_; } + int local_files(int i) const { return local_files_[i]; } + int local_thumbnail_files() const { return local_files_[0]; } + int local_medium_files() const { return local_files_[1]; } + int local_full_files() const { return local_files_[2]; } + int local_original_files() const { return local_files_[3]; } + + private: + PhotoStorage* const photo_storage_; + Mutex mu_; + ScopedPtr iter_; + const string prefix_; + Slice pos_; + int64_t local_bytes_; + int local_files_[4]; + }; + + friend class Scanner; + typedef std::unordered_set ScannerSet; + + // A structure containing a local storage setting (value, title string and + // detail string) for use by SettingViewController. + struct Setting { + Setting(int64_t v = 0, + const string& t = string(), + const string& d = string()) + : value(v), + title(t), + detail(d) { + } + int64_t value; + string title; + string detail; + }; + + public: + PhotoStorage(AppState* state); + ~PhotoStorage(); + + // Writes the data to the specified file, updating the md5 and access time + // for the file. The parent_size indicates the size (thumbnail, medium, full, + // etc) of the image which generated the new image. A parent_size of 0 + // indicates the image was generated from an ALAsset. + bool Write(const string& filename, int parent_size, + const Slice& data, const DBHandle& updates); + // Adds an existing file (specified by path) to the photo storage with the + // name "filename", updating the md5 and access time for the file. + bool AddExisting(const string& path, const string& filename, + const string& md5, const string& server_id, + const DBHandle& updates); + // Sets the server id associated with filename, renaming the file on disk and + // storing a level of indirection in the database so that we can still access + // the image via "filename". + void SetServerId(const string& filename, const string& server_id, + const DBHandle& updates); + // Add a symlink for server_id to asset_key in the photo server directory. + void SetAssetSymlink(const string& filename, const string& server_id, + const string& asset_key); + // Read the asset symlink for server_id. + string ReadAssetSymlink(const string& filename, const string& server_id); + // Returns true if we have uploaded the asset for the specified server id. + bool HaveAssetSymlink(const string& filename, const string& server_id, + const string& asset_key); + // Deletes the specified file and the md5 and access time information. + void Delete(const string& filename, const DBHandle& updates); + // Delete all of the files associated with the photo id. + void DeleteAll(int64_t photo_id, const DBHandle& updates); + // Reads the data from the specified file, updating the access time for the + // file. + string Read(const string& filename, const string& metadata); + // Updates the access time for the specified file. + void Touch(const string& filename, const string& metadata); + // Returns true if the file exists or if the server file exists. If only the + // server file exists, the local file is linked to the server file and the + // metadata is populated. + bool MaybeLinkServerId(const string& filename, const string& server_id, + const string& md5, const DBHandle& updates); + // Returns true if the file exists, false otherwise. + bool Exists(const string& filename); + // Returns the size of the specified file. + int64_t Size(const string& filename); + // Returns the metadata for the specified file. + PhotoPathMetadata Metadata(const string& filename); + + // Returns the smallest resolution image that is greater than or equal to + // max_size. The raw (unparsed) metadata for the filename is returned in the + // metadata parameter, suitable for passing to Read(). + string LowerBound(int64_t photo_id, int max_size, string* metadata); + + string PhotoPath(const Slice& filename); + + // Garbage collect files that do not have a corresponding entry in the + // database. + void GarbageCollect(); + + // Check the photo storage metadata for consistency. Returns true if the + // internal consistency is ok and false otherwise. + bool Check(); + + void set_local_bytes_limit(int64_t v); + int64_t local_bytes_limit() const { return local_bytes_limit_; } + int64_t local_bytes() const { return local_bytes_; } + int local_thumbnail_files() const { return local_files_[0]; } + int local_medium_files() const { return local_files_[1]; } + int local_full_files() const { return local_files_[2]; } + int local_original_files() const { return local_files_[3]; } + const vector& settings() const; + int setting_index(int64_t value) const; + CallbackSet* changed() { return &changed_; } + // Merge current usage with the passed in one (not all categories may be filled). + void update_remote_usage(const UsageMetadata& usage); + UsageMetadata remote_usage() const; + + private: + string PhotoServerPath(const Slice& filename, const string& server_id); + void IncrementLocalUsage(const string& filename, + int64_t deleta, const DBHandle& updates); + void FixLocalUsage(); + void AddScanner(Scanner* scanner); + void RemoveScanner(Scanner* scanner); + void AddUncommittedFile(const string& filename); + void RemoveUncommittedFile(const string& filename); + bool IsUncommittedFile(const string& filename); + void CommonInit(); + + private: + AppState* const state_; + ScopedPtr async_; + const string photo_dir_; + const string server_photo_dir_; + mutable Mutex mu_; + CallbackSet changed_; + StringSet uncommitted_files_; + int64_t local_bytes_limit_; + int64_t local_bytes_; + int local_files_[4]; + bool gc_; + ScannerSet scanners_; + UsageMetadata remote_usage_; +}; + +#endif // VIEWFINDER_PHOTO_STORAGE_H diff --git a/clients/shared/PhotoTable.cc b/clients/shared/PhotoTable.cc new file mode 100644 index 0000000..158c706 --- /dev/null +++ b/clients/shared/PhotoTable.cc @@ -0,0 +1,1095 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. +// +// PhotoTable maintain the following tables for assets: +// +// # -> +// # -> +// +// When an asset is encountered during a scan, we either create a new photo, or +// add the above mappings pointing to an empty string. When we notice an asset +// has been deleted we update the corresponding PhotoMetadata to remove the +// asset-url but leave the asset-fingerprint in place. + +#import +#import "AppState.h" +#import "AsyncState.h" +#import "DayTable.h" +#import "GeocodeManager.h" +#import "ImageIndex.h" +#import "LazyStaticPtr.h" +#import "LocationUtils.h" +#import "NetworkQueue.h" +#import "PhotoStorage.h" +#import "PhotoTable.h" +#import "PlacemarkHistogram.h" +#import "PlacemarkTable.h" +#import "ServerUtils.h" +#import "StringUtils.h" +#import "Timer.h" +#import "WallTime.h" + +const string PhotoTable::kPhotoDuplicateQueueKeyPrefix = DBFormat::photo_duplicate_queue_key(); + +namespace { + +const int kPhotoFSCKVersion = 3; +const int kUnquarantineVersion = 4; + +const int kSecondsInHour = 60 * 60; +const int kSecondsInDay = kSecondsInHour * 24; + +const WallTime kURLExpirationSlop = 60; +const string kAssetFingerprintKeyPrefix = DBFormat::asset_fingerprint_key(""); +const string kDeprecatedAssetReverseKeyPrefix = DBFormat::deprecated_asset_reverse_key(""); +const string kUnquarantineVersionKey = DBFormat::metadata_key("unquarantine_version"); + +const string kPerceptualFingerprintPrefix = "P"; +const int kPerceptualFingerprintBinarySize = 20; +const int kPerceptualFingerprintSize = 29; + +// S3 urls look like: +// https://s3/foo?Signature=bar&Expires=1347112861&AWSAccessKeyId=blah +LazyStaticPtr kS3URLRE = { + ".*[?&]Expires=([0-9]+).*" +}; + +const DBRegisterKeyIntrospect kPhotoKeyIntrospect( + DBFormat::photo_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kPhotoDuplicateQueueKeyIntrospect( + PhotoTable::kPhotoDuplicateQueueKeyPrefix, + [](Slice key) { + int64_t local_id; + if (!DecodePhotoDuplicateQueueKey(key, &local_id)) { + return string(); + } + return string(Format("%d", local_id)); + }, NULL); + +const DBRegisterKeyIntrospect kPhotoServerKeyIntrospect( + DBFormat::photo_server_key(), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kPhotoURLKey( + DBFormat::photo_url_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kAssetKeyIntrospect( + DBFormat::asset_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kAssetDeprecatedReverseKeyIntrospect( + DBFormat::deprecated_asset_reverse_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kAssetFingerprintKeyIntrospect( + DBFormat::asset_fingerprint_key(""), NULL, [](Slice value) { + return value.ToString(); + }); + +string EncodePhotoURLKey(int64_t id, const string& name) { + return DBFormat::photo_url_key(Format("%d/%s", id, name)); +} + +} // namespace + +string EncodeAssetFingerprintKey(const Slice& fingerprint) { + string s = kAssetFingerprintKeyPrefix; + fingerprint.AppendToString(&s); + return s; +} + +string EncodePhotoDuplicateQueueKey(int64_t local_id) { + string s = PhotoTable::kPhotoDuplicateQueueKeyPrefix; + OrderedCodeEncodeVarint64(&s, local_id); + return s; +} + +string EncodePerceptualFingerprint(const Slice& term) { + DCHECK_EQ(kPerceptualFingerprintBinarySize, term.size()); + return kPerceptualFingerprintPrefix + Base64Encode(term); +} + +bool DecodeAssetFingerprintKey(Slice key, Slice* fingerprint) { + if (!key.starts_with(kAssetFingerprintKeyPrefix)) { + return false; + } + key.remove_prefix(kAssetFingerprintKeyPrefix.size()); + *fingerprint = key; + return true; +} + +bool DecodePhotoDuplicateQueueKey(Slice key, int64_t* local_id) { + if (!key.starts_with(PhotoTable::kPhotoDuplicateQueueKeyPrefix)) { + return false; + } + key.remove_prefix(PhotoTable::kPhotoDuplicateQueueKeyPrefix.size()); + *local_id = OrderedCodeDecodeVarint64(&key); + return true; +} + +bool DecodePerceptualFingerprint(Slice fingerprint, string* term) { + if (fingerprint.size() != kPerceptualFingerprintSize || + !fingerprint.starts_with(kPerceptualFingerprintPrefix)) { + return false; + } + fingerprint.remove_prefix(kPerceptualFingerprintPrefix.size()); + if (term) { + *term = Base64Decode(fingerprint); + } + return true; +} + +bool DecodeDeprecatedAssetReverseKey(Slice key, Slice* fingerprint, Slice* url) { + if (!key.starts_with(kDeprecatedAssetReverseKeyPrefix)) { + return false; + } + key.remove_prefix(kDeprecatedAssetReverseKeyPrefix.size()); + const int pos = key.rfind('#'); + if (pos == key.npos) { + return false; + } + if (fingerprint) { + *fingerprint = key.substr(0, pos); + } + if (url) { + *url = key.substr(pos + 1); + } + return true; +} + +PhotoTable_Photo::PhotoTable_Photo(AppState* state, const DBHandle& db, int64_t id) + : state_(state), + db_(db) { + mutable_id()->set_local_id(id); +} + +void PhotoTable_Photo::MergeFrom(const PhotoMetadata& m) { + // Some assertions that immutable properties don't change. + if (episode_id().has_server_id() && m.episode_id().has_server_id()) { + DCHECK_EQ(episode_id().server_id(), m.episode_id().server_id()); + } + if (has_user_id() && m.has_user_id()) { + DCHECK_EQ(user_id(), m.user_id()); + } + if (has_timestamp() && m.has_timestamp()) { + // TODO(peter): I have photos in my asset library with timestamps that + // differ by more than a second from the data stored on the server. + // DCHECK_EQ(trunc(timestamp()), trunc(m.timestamp())); + } + + PhotoMetadata::MergeFrom(m); +} + +void PhotoTable_Photo::MergeFrom(const ::google::protobuf::Message&) { + DIE("MergeFrom(Message&) should not be used"); +} + + +int64_t PhotoTable_Photo::GetDeviceId() const { + if (!id().has_server_id()) { + return state_->device_id(); + } + int64_t device_id = 0; + int64_t dummy_id = 0; + WallTime dummy_timestamp = 0; + DecodePhotoId( + id().server_id(), &device_id, &dummy_id, &dummy_timestamp); + return device_id; +} + +int64_t PhotoTable_Photo::GetUserId() const { + return has_user_id() ? user_id() : state_->user_id(); +} + +bool PhotoTable_Photo::GetLocation(Location* loc, Placemark* pm) { + if (has_location() && loc) { + loc->CopyFrom(location()); + } + // If we have a location, but the placemark isn't set, try to + // reverse geocode. + // NOTE: there are some photos which have location and placemark + // is set, but is empty. Make sure we reverse geocode in this case. + if (has_location() && (!has_placemark() || !placemark().has_country())) { + state_->photo_table()->MaybeReverseGeocode(local_id()); + return true; + } + if (has_placemark() && pm) { + pm->CopyFrom(placemark()); + } + return has_location(); +} + +string PhotoTable_Photo::FormatLocation(bool shorten) { + Location location; + Placemark placemark; + if (GetLocation(&location, &placemark)) { + string s; + state_->placemark_histogram()->FormatLocation(location, placemark, shorten, &s); + return s; + } + return shorten ? "" : "Location Unavailable"; +} + +string PhotoTable_Photo::GetURL(const string& name) { + return db_->Get(EncodePhotoURLKey(local_id(), name)); +} + +string PhotoTable_Photo::GetUnexpiredURL( + const string& name, const DBHandle& updates) { + const string url = GetURL(name); + if (!url.empty()) { + int expires = 0; + if (!RE2::FullMatch(url, *kS3URLRE, &expires)) { + return url; + } + if (expires >= WallTime_Now() + kURLExpirationSlop) { + return url; + } + DeleteURL(name, updates); + } + return string(); +} + +void PhotoTable_Photo::SetURL( + const string& name, const string& url, const DBHandle& updates) { + updates->Put(EncodePhotoURLKey(local_id(), name), url); +} + +void PhotoTable_Photo::DeleteURL(const string& name, const DBHandle& updates) { + updates->Delete(EncodePhotoURLKey(local_id(), name)); +} + +bool PhotoTable_Photo::ShouldUpdateTimestamp(WallTime exif_timestamp) { + const int64_t orig_seconds = trunc(timestamp()); + const int64_t exif_seconds = trunc(exif_timestamp); + + if (fabs(orig_seconds - exif_seconds) < kSecondsInDay && + (orig_seconds % kSecondsInHour) == (exif_seconds % kSecondsInHour)) { + LOG("photo: %s: original (%s) and exif (%s) timestamps [probably] differ as " + "time zone information is not captured in exif data; ignoring difference " + "and continuing with original", id(), + WallTimeFormat("%F %T", timestamp()), + WallTimeFormat("%F %T", exif_timestamp)); + return false; + } + + return true; +} + +void PhotoTable_Photo::Quarantine( + const string& reason, const DBHandle& updates) { + LOG("photo: quarantining %s: %s", id(), reason); + + // Mark the photo as quarantined in the database. This will prevent the photo + // from reappearing the next time the app starts. + set_label_error(true); + + // Remove the photo from every episode it is posted to. This causes it to + // disappear in the UI. + vector episode_ids; + state_->episode_table()->ListEpisodes(local_id(), &episode_ids, updates); + for (int i = 0; i < episode_ids.size(); ++i) { + EpisodeHandle e = state_->episode_table()->LoadEpisode(episode_ids[i], updates); + e->Lock(); + e->QuarantinePhoto(local_id()); + e->SaveAndUnlock(updates); + } +} + +void PhotoTable_Photo::Invalidate(const DBHandle& updates) { + vector episode_ids; + state_->episode_table()->ListEpisodes(local_id(), &episode_ids, updates); + for (int i = 0; i < episode_ids.size(); ++i) { + EpisodeHandle e = state_->episode_table()->LoadEpisode(episode_ids[i], updates); + if (e.get()) { + e->Invalidate(updates); + } + } +} + +bool PhotoTable_Photo::Load() { + disk_asset_keys_ = GetAssetKeySet(); + disk_perceptual_fingerprint_ = perceptual_fingerprint(); + day_table_fields_ = GetDayTableFields(); + return true; +} + +void PhotoTable_Photo::SaveHook(const DBHandle& updates) { + StringSet asset_keys_set = GetAssetKeySet(); + for (StringSet::iterator it = disk_asset_keys_.begin(); it != disk_asset_keys_.end(); ++it) { + if (!ContainsKey(asset_keys_set, *it)) { + updates->Delete(*it); + } + } + disk_asset_keys_ = asset_keys_set; + + for (int i = 0; i < asset_keys_size(); i++) { + updates->Put(asset_keys(i), local_id()); + } + + if (disk_perceptual_fingerprint_.SerializeAsString() != + perceptual_fingerprint().SerializeAsString()) { + const string id_str = ToString(id().local_id()); + state_->image_index()->Remove(disk_perceptual_fingerprint_, id_str, updates); + disk_perceptual_fingerprint_ = perceptual_fingerprint(); + state_->image_index()->Add(disk_perceptual_fingerprint_, id_str, updates); + + for (int i = 0; i < disk_perceptual_fingerprint_.terms_size(); ++i) { + AddAssetFingerprint(EncodePerceptualFingerprint( + disk_perceptual_fingerprint_.terms(i)), false); + } + } + + for (int i = 0; i < asset_fingerprints_size(); i++) { + updates->Put(EncodeAssetFingerprintKey(asset_fingerprints(i)), local_id()); + } + + // The "removed" and "unshared" labels affect the post relationship between + // an episode and a photo. They are only used in client/server communication + // and should not be persisted to disk. + clear_label_removed(); + clear_label_unshared(); + + if (has_location() && has_placemark() && !placemark_histogram()) { + set_placemark_histogram(true); + state_->placemark_histogram()->AddPlacemark( + placemark(), location(), updates); + } + + const string new_day_table_fields = GetDayTableFields(); + if (day_table_fields_ != new_day_table_fields) { + // Only invalidate if the day table fields have changed. Note that we don't + // have to invalidate the episodes if the day table fields are empty, which + // indicates that the photo did not previously exist. If the photo was just + // created the episode it was added to (if any) will have already been + // invalidated. + if (!day_table_fields_.empty()) { + // Invalidate all activities which have shared this photo and all episodes + // which contain it. + Invalidate(updates); + } + day_table_fields_ = new_day_table_fields; + } + + // Ugh, PhotoTable_Photo is the base class but PhotoHandle needs a pointer to + // the superclass. + typedef ContentTable::Content Content; + Content* content = reinterpret_cast(this); + state_->net_queue()->QueuePhoto(PhotoHandle(content), updates); + + if (!label_error() && candidate_duplicates_size() > 0) { + updates->Put(EncodePhotoDuplicateQueueKey(local_id()), string()); + AppState* s = state_; + updates->AddCommitTrigger("PhotoDuplicateQueue", [s] { + s->ProcessPhotoDuplicateQueue(); + }); + } else { + updates->Delete(EncodePhotoDuplicateQueueKey(local_id())); + } +} + +void PhotoTable_Photo::DeleteHook(const DBHandle& updates) { + for (StringSet::iterator it = disk_asset_keys_.begin(); it != disk_asset_keys_.end(); ++it) { + updates->Delete(*it); + } + + if (disk_perceptual_fingerprint_.terms_size() > 0) { + const string id_str = ToString(id().local_id()); + state_->image_index()->Remove(disk_perceptual_fingerprint_, id_str, updates); + } + + if (has_location() && has_placemark() && placemark_histogram()) { + clear_placemark_histogram(); + state_->placemark_histogram()->RemovePlacemark( + placemark(), location(), updates); + } + + // Ugh, PhotoTable_Photo is the base class but PhotoHandle needs a pointer to + // the superclass. + typedef ContentTable::Content Content; + Content* content = reinterpret_cast(this); + state_->net_queue()->DequeuePhoto(PhotoHandle(content), updates); + + updates->Delete(EncodePhotoDuplicateQueueKey(local_id())); + + state_->photo_storage()->DeleteAll(local_id(), updates); +} + +StringSet PhotoTable_Photo::GetAssetKeySet() const { + return StringSet(asset_keys().begin(), asset_keys().end()); +} + +string PhotoTable_Photo::GetDayTableFields() const { + PhotoMetadata m; + if (has_id()) { + m.mutable_id()->CopyFrom(id()); + // The server-id for a photo is set the first time the photo is loaded and + // its original timestamp is verified. We don't need to perform a day table + // refresh when that occurs. + m.mutable_id()->clear_server_id(); + } + if (has_episode_id()) { + m.mutable_episode_id()->CopyFrom(episode_id()); + } + if (has_placemark()) { + m.mutable_placemark()->CopyFrom(placemark()); + } + if (has_aspect_ratio()) { + m.set_aspect_ratio(aspect_ratio()); + } + if (has_timestamp()) { + m.set_timestamp(timestamp()); + } + return m.SerializeAsString(); +} + +bool PhotoTable_Photo::ShouldAddPhotoToEpisode() const { + if (!has_aspect_ratio() || + std::isnan(aspect_ratio()) || + !has_timestamp() || + (candidate_duplicates_size() > 0)) { + // We don't have enough photo metadata to match the photo to an + // episode. Note that we'll add a photo to an episode before we've + // downloaded any of the photo images and rely on the prioritization of + // images needed for the UI. + return false; + } + return true; +} + +void PhotoTable_Photo::FindCandidateDuplicates() { + if (!has_perceptual_fingerprint()) { + return; + } + WallTimer timer; + StringSet matched_ids; + state_->image_index()->Search(state_->db(), perceptual_fingerprint(), &matched_ids); + clear_candidate_duplicates(); + for (StringSet::iterator iter(matched_ids.begin()); + iter != matched_ids.end(); + ++iter) { + const int64_t local_id = FromString(*iter); + if (local_id == id().local_id()) { + // Never consider a photo its own candidate duplicate. + continue; + } + add_candidate_duplicates(local_id); + } + if (candidate_duplicates_size() > 0) { + LOG("photo: %s: %d candidate duplicates (%.2f ms): %s", + id(), candidate_duplicates_size(), timer.Milliseconds(), matched_ids); + } +} + +bool PhotoTable_Photo::HasAssetUrl() const { + return asset_keys_size() > 0; +} + +bool PhotoTable_Photo::AddAssetKey(const string& asset_key) { + Slice url; + Slice fingerprint; + if (!DecodeAssetKey(asset_key, &url, &fingerprint)) { + LOG("photo: invalid asset key %s", asset_key); + return false; + } + + bool changed = false; + if (!fingerprint.empty()) { + changed = AddAssetFingerprint(fingerprint, false); + } + + DCHECK(!url.empty()); // Shouldn't happen any more, but just in case. + if (!url.empty()) { + bool found = false; + for (int i = 0; i < asset_keys_size(); i++) { + if (asset_keys(i) == asset_key) { + found = true; + break; + } + + Slice url2, fingerprint2; + if (!DecodeAssetKey(asset_keys(i), &url2, &fingerprint2)) { + continue; + } + if (url == url2 && !fingerprint.empty() && fingerprint2.empty()) { + // Upgrade the existing url-only asset key to include the fingerprint. + set_asset_keys(i, asset_key); + changed = true; + found = true; + } + } + if (!found) { + add_asset_keys(asset_key); + changed = true; + } + } + + return changed; +} + +bool PhotoTable_Photo::AddAssetFingerprint(const Slice& fingerprint, bool from_server) { + for (int i = 0; i < asset_fingerprints_size(); i++) { + if (asset_fingerprints(i) == fingerprint) { + return false; + } + } + add_asset_fingerprints(fingerprint.as_string()); + // If the server doesn't know about this fingerprint, upload the metadata. + if (!from_server) { + set_update_metadata(true); + } else { + string term; + if (DecodePerceptualFingerprint(fingerprint, &term)) { + ImageFingerprint* pf = mutable_perceptual_fingerprint(); + for (int i = 0; i < pf->terms_size(); i++) { + if (pf->terms(i) == term) { + return true; + } + } + pf->add_terms(term); + } + } + return true; +} + +bool PhotoTable_Photo::RemoveAssetKey(const string& asset_key) { + for (int i = 0; i < asset_keys_size(); i++) { + if (asset_keys(i) == asset_key) { + RemoveAssetKeyByIndex(i); + return true; + } + } + return false; +} + +bool PhotoTable_Photo::RemoveAssetKeyByIndex(int index) { + DCHECK_GE(index, 0); + DCHECK_LT(index, asset_keys_size()); + if (index < 0 || index >= asset_keys_size()) { + return false; + } + ProtoRepeatedFieldRemoveElement(mutable_asset_keys(), index); + return true; +} + +bool PhotoTable_Photo::InLibrary() { + return state_->episode_table()->ListLibraryEpisodes( + id().local_id(), NULL, db_); +} + +bool PhotoTable_Photo::MaybeSetServerId() { + const int64_t device_id = state_->device_id(); + if (id().has_server_id() || !device_id) { + return false; + } + mutable_id()->set_server_id( + EncodePhotoId(device_id, id().local_id(), timestamp())); + return true; +} + +PhotoTable::PhotoTable(AppState* state) + : ContentTable(state, + DBFormat::photo_key(), + DBFormat::photo_server_key(), + kPhotoFSCKVersion, + DBFormat::metadata_key("photo_table_fsck")), + geocode_in_progress_(0) { +} + +PhotoTable::~PhotoTable() { +} + +void PhotoTable::Reset() { +} + +PhotoHandle PhotoTable::LoadPhoto(const PhotoId& id, const DBHandle& db) { + PhotoHandle ph; + if (id.has_local_id()) { + ph = LoadPhoto(id.local_id(), db); + } + if (!ph.get() && id.has_server_id()) { + ph = LoadPhoto(id.server_id(), db); + } + return ph; +} + +PhotoHandle PhotoTable::LoadAssetPhoto( + const Slice& asset_key, const DBHandle& db) { + const int64_t id = AssetToDeviceId(asset_key, false, db); + if (id == -1) { + return ContentHandle(); + } + return LoadPhoto(id, db); +} + +bool PhotoTable::AssetPhotoExists( + const Slice& url, const Slice& fingerprint, const DBHandle& db) { + return AssetToDeviceId(url, fingerprint, true, db) != -1; +} + +bool PhotoTable::AssetPhotoExists( + const Slice& asset_key, const DBHandle& db) { + return AssetToDeviceId(asset_key, true, db) != -1; +} + +void PhotoTable::AssetsNotFound( + const StringSet& not_found, const DBHandle& updates) { + for (StringSet::const_iterator iter(not_found.begin()); + iter != not_found.end(); + ++iter) { + const string& url = *iter; + DCHECK(!url.empty()); + if (url.empty()) { + // TODO(peter): Perhaps do something slightly more encompassing as far as + // validation. A super short prefix could still remove a bunch of stuff, + // though asset-urls currently appear to be a fixed length (78 bytes?). + continue; + } + + // Loop over all (there is normally only 1, but might be more if this is + // the first full scan after asset urls have changed) of the asset-keys + // with url as a prefix. + for (DB::PrefixIterator iter(updates, EncodeAssetKey(url, "")); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + Slice fingerprint; + if (!DecodeAssetKey(key, NULL, &fingerprint)) { + // This shouldn't happen. + DCHECK(false) << ": unable to decode: " << key; + continue; + } + + // Load the associated photo and clear the url from the asset key. + PhotoHandle ph = LoadAssetPhoto(key, updates); + if (ph.get()) { + ph->Lock(); + // Clear the url, but leave the fingerprint. + ph->RemoveAssetKey(key.as_string()); + if (ph->upload_metadata() && !ph->HasAssetUrl()) { + // The photo has not been uploaded to the server and only exists + // locally. The local asset has disappeared. Quarantine. + ph->Quarantine("garbage collect", updates); + } + ph->SaveAndUnlock(updates); + } else { + // No asset associated with the key. Nothing to update, just + // delete. + updates->Delete(key); + if (!fingerprint.empty()) { + updates->Delete(EncodeAssetFingerprintKey(fingerprint)); + } + } + } + } +} + +void PhotoTable::DeleteAllImages(int64_t photo_id, const DBHandle& updates) { + const PhotoHandle ph = LoadPhoto(photo_id, updates); + LOG("photo table: deleting all images for photo %d", photo_id); + if (ph.get()) { + // Clear the download bits: the photo won't ever be displayed in the UI, so + // there is no need to download any images. + ph->Lock(); + ph->clear_download_thumbnail(); + ph->clear_download_full(); + ph->clear_download_medium(); + ph->clear_download_original(); + // Clear asset keys from photo metadata, delete from index, and try + // to delete underlying assets. + for (int i = 0; i < ph->asset_keys_size(); i++) { + const string key = ph->asset_keys(i); + // Clear the url, but leave the fingerprint. + ph->RemoveAssetKey(key); + updates->Delete(key); + state_->DeleteAsset(key); + } + + ph->SaveAndUnlock(updates); + } + state_->photo_storage()->DeleteAll(photo_id, updates); +} + +bool PhotoTable::PhotoInLibrary(int64_t photo_id, const DBHandle& db) { + PhotoHandle ph = LoadPhoto(photo_id, db); + return ph.get() && ph->InLibrary(); +} + +bool PhotoTable::IsAssetPhotoEqual( + const PhotoMetadata& local, const PhotoMetadata& server) { + std::set local_fingerprints; + std::set server_fingerprints; + for (int i = 0; i < local.asset_fingerprints_size(); ++i) { + local_fingerprints.insert(local.asset_fingerprints(i)); + } + for (int i = 0; i < server.asset_fingerprints_size(); ++i) { + server_fingerprints.insert(server.asset_fingerprints(i)); + } + + if (!server_fingerprints.empty()) { + // If we have an asset fingerprint on the server, only consider the two + // photos equal if we have the matching fingerprint locally. + // All photos uploaded since version 1.2 have fingerprints. + return SetsIntersect(local_fingerprints, server_fingerprints); + } + + // If we don't have fingerprints, use some crude heuristics to tell if the photo + // is definitely different, otherwise assume nothing has happened that would + // cause asset ids to be reused. + // Unfortunately, most metadata fields are unreliable for this purpose. + // Photos can losslessly have their orientation fields normalized, and iTunes + // appears to sometimes alter timestamps based on the current local timezone. + + if (local.has_location() != server.has_location()) { + // The local photo has a location while the server photo does not (or vice + // versa). These cannot be the same photo. + return false; + } + if (local.has_location()) { + if (DistanceBetweenLocations(local.location(), server.location()) > .1) { + // The local and server photo locations differ too much. These cannot be + // the same photo. + return false; + } + } + return true; +} + +bool PhotoTable::ReverseGeocode( + int64_t photo_id, GeocodeCallback completion) { + // This must always be running on the main thread. + DCHECK(dispatch_is_main_thread()); + if (!dispatch_is_main_thread()) { + LOG("cannot reverse geocode except on main thread; this should never happen"); + return false; + } + if (!state_->geocode_manager()) { + return false; + } + PhotoHandle p = LoadPhoto(photo_id, state_->db()); + if (!p.get()) { + LOG("photo: %s is not a valid photo id", photo_id); + return false; + } + // Exit if: + // - No location set. + // - Invalid location. + // - A previous error with placemark was encountered. + // - We already have a valid placemark. + if (!p->has_location() || + !PlacemarkTable::IsLocationValid(p->location()) || + p->error_placemark_invalid() || + (p->has_placemark() && PlacemarkTable::IsPlacemarkValid(p->placemark()))) { + return false; + } + VLOG("reverse geocode on location: %s", p->location()); + + PlacemarkHandle h = state_->placemark_table()->FindPlacemark(p->location(), state_->db()); + if (h->valid()) { + // Optimized the reverse geocode out of existence by reusing the placemark + // for a previously reverse geocoded photo at the same location. + LOG("reverse geocode photo %d with location: %s, placemark: %s", + p->id().local_id(), p->location(), *h); + p->Lock(); + p->mutable_placemark()->CopyFrom(*h); + DBHandle updates = state_->NewDBTransaction(); + p->SaveAndUnlock(updates); + updates->Commit(); + return false; + } + + // Look up the callback set for this location. + const Location* l = &h->location(); + GeocodeCallbackSet* callbacks = geocode_callback_map_[l]; + const bool do_reverse_geocode = !callbacks; + if (!callbacks) { + callbacks = new GeocodeCallbackSet; + geocode_callback_map_[l] = callbacks; + } + // Add the completion to the callback set. + callbacks->Add(completion); + + if (do_reverse_geocode) { + // Only run reverse geocoding for the first caller interested in reverse + // geocoding this location. + CHECK(state_->geocode_manager()->ReverseGeocode( + l, [this, p, h, l, callbacks](const Placemark* m) { + DBHandle updates = state_->NewDBTransaction(); + p->Lock(); + if (m && PlacemarkTable::IsPlacemarkValid(*m)) { + h->Lock(); + h->CopyFrom(*m); + h->SaveAndUnlock(updates); + p->mutable_placemark()->CopyFrom(*m); + } else if (m) { + // On an invalid placemark, indicate a placemark error so we don't retry. + p->set_error_placemark_invalid(true); + } else { + p->mutable_placemark()->Clear(); + } + p->SaveAndUnlock(updates); + updates->Commit(); + // Erase from geocode_callback_map_ before running the + // callbacks because running the callbacks might want to + // reverse geocode the same placemark again if geocoding + // failed. + geocode_callback_map_.erase(l); + callbacks->Run(p->has_placemark()); + delete callbacks; + })); + } else { + CHECK_GT(callbacks->size(), 1); + } + return true; +} + +void PhotoTable::MaybeUnquarantinePhoto( + const PhotoHandle& ph, const DBHandle& updates) { + vector episode_ids; + state_->episode_table()->ListEpisodes(ph->id().local_id(), &episode_ids, updates); + if (episode_ids.empty()) { + // Try to load original episode. + episode_ids.push_back(ph->episode_id().local_id()); + } + int count = 0; + for (int i = 0; i < episode_ids.size(); ++i) { + EpisodeHandle eh = state_->episode_table()->LoadEpisode(episode_ids[i], updates); + if (eh.get()) { + eh->Lock(); + eh->AddPhoto(ph->id().local_id()); + eh->SaveAndUnlock(updates); + ++count; + } + } + if (!count) { + LOG("photo unquarantine: photo %s had no episodes", *ph); + return; + } + LOG("photo unquarantine: resetting error bit on photo %s and " + "re-posting to %d episodes", *ph, count); + ph->Lock(); + ph->clear_label_error(); + if (ph->error_upload_thumbnail()) { + // If we encountered an error uploading the thumbnail, try + // uploading the metadata again. + ph->set_upload_metadata(true); + } + ph->SaveAndUnlock(updates); +} + +void PhotoTable::MaybeUnquarantinePhoto(int64_t photo_id) { + DBHandle updates = state_->NewDBTransaction(); + PhotoHandle ph = LoadPhoto(photo_id, updates); + MaybeUnquarantinePhoto(ph, updates); + updates->Commit(); +} + +bool PhotoTable::MaybeUnquarantinePhotos(ProgressUpdateBlock progress_update) { + const int cur_version = state_->db()->Get(kUnquarantineVersionKey, 0); + if (cur_version >= kUnquarantineVersion) { + return false; + } + + if (progress_update) { + progress_update("Reviving Missing Photos"); + } + + int unquarantines = 0; + DBHandle updates = state_->NewDBTransaction(); + for (DB::PrefixIterator iter(updates, DBFormat::photo_key()); + iter.Valid(); + iter.Next()) { + const int64_t ph_id = DecodeContentKey(iter.key()); + PhotoHandle ph = LoadPhoto(ph_id, updates); + if (ph->label_error()) { + MaybeUnquarantinePhoto(ph, updates); + ++unquarantines; + } + } + + updates->Put(kUnquarantineVersionKey, kUnquarantineVersion); + updates->Commit(); + return unquarantines > 0; +} + +void PhotoTable::MaybeReverseGeocode(int64_t photo_id) { + MutexLock l(&mu_); + if (geocode_in_progress_) { + return; + } + const GeocodeCallback completion = [this](bool success) { + MutexLock l(&mu_); + --geocode_in_progress_; + }; + // Need to run this on a different thread as we can arrive at this + // point with locks held and PhotoManager::ReverseGeocode may invoke + // its callback synchronously in case the requested location is in + // the placemark table. + state_->async()->dispatch_after_main( + 0, [this, photo_id, completion] { + if (ReverseGeocode(photo_id, completion)) { + ++geocode_in_progress_; + } + }); +} + +int64_t PhotoTable::AssetToDeviceId( + const Slice& url, const Slice& fingerprint, + bool require_url_match, const DBHandle& db) { + // Look up the existing mapping for the url. + int64_t local_id = -1; + if (!url.empty()) { + for (DB::PrefixIterator iter(db, EncodeAssetKey(url, "")); + iter.Valid(); + iter.Next()) { + Slice existing_url; + Slice existing_fingerprint; + if (!DecodeAssetKey(iter.key(), &existing_url, &existing_fingerprint)) { + // This shouldn't happen. + DCHECK(false) << ": unable to decode: " << iter.key(); + continue; + } + if (require_url_match && url != existing_url) { + continue; + } + if (fingerprint.empty() || + existing_fingerprint.empty() || + fingerprint == existing_fingerprint) { + local_id = FromString(iter.value(), -1); + } + break; + } + } + + if (local_id == -1 && !require_url_match && !fingerprint.empty()) { + local_id = db->Get(EncodeAssetFingerprintKey(fingerprint), -1); + } + return local_id; +} + +int64_t PhotoTable::AssetToDeviceId( + const Slice& asset_key, bool require_url_match, + const DBHandle& db) { + Slice url; + Slice fingerprint; + if (!DecodeAssetKey(asset_key, &url, &fingerprint)) { + return false; + } + return AssetToDeviceId(url, fingerprint, require_url_match, db); +} + +bool PhotoTable::FSCKImpl(int prev_fsck_version, const DBHandle& updates) { + LOG("FSCK: PhotoTable"); + bool changes = false; + + for (DB::PrefixIterator iter(updates, DBFormat::photo_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + PhotoMetadata pm; + if (pm.ParseFromArray(value.data(), value.size())) { + PhotoHandle ph = LoadPhoto(pm.id().local_id(), updates); + ph->Lock(); + bool save_ph = false; + if (key != EncodeContentKey(DBFormat::photo_key(), pm.id().local_id())) { + LOG("FSCK: photo id %d does not equal key %s; deleting key and re-saving", + pm.id().local_id(), key); + updates->Delete(key); + save_ph = true; + } + + // Check server key mapping. + if (ph->id().has_server_id()) { + const string server_key = EncodeContentServerKey(DBFormat::photo_server_key(), + ph->id().server_id()); + if (!updates->Exists(server_key)) { + LOG("FSCK: missing photo server key mapping"); + save_ph = true; + } else { + const int64_t mapped_local_id = updates->Get(server_key, -1); + if (mapped_local_id != ph->id().local_id()) { + LOG("FSCK: photo local id mismatch: %d != %d; deleting existing mapping", + mapped_local_id, ph->id().local_id()); + updates->Delete(server_key); + save_ph = true; + } + } + } + + if (ph->shared() && ph->upload_medium()) { + // Re-save any shared with with a non-uploaded medium resolution + // image to force it to be re-added to the network queue. + LOG("FSCK: non-uploaded medium resolution image"); + save_ph = true; + } + + // Check the asset key index. + const string local_id_str = ToString(ph->id().local_id()); + for (int i = 0; i < ph->asset_keys_size(); i++) { + Slice url, fingerprint; + if (!DecodeAssetKey(ph->asset_keys(i), &url, &fingerprint)) { + continue; + } + + string value; + if (!updates->Get(ph->asset_keys(i), &value)) { + LOG("FSCK: adding missing asset key index %s for photo %s", ph->asset_keys(i), local_id_str); + updates->Put(ph->asset_keys(i), local_id_str); + changes = true; + } else if (value != local_id_str) { + LOG("FSCK: removing conflicting asset key %s from photo %s (owned by %s)", ph->asset_keys(i), + local_id_str, value); + // Remove it from disk_asset_keys so the save doesn't remove the entry from its owner. + ph->disk_asset_keys_.erase(ph->asset_keys(i)); + ProtoRepeatedFieldRemoveElement(ph->mutable_asset_keys(), i); + save_ph = true; + --i; + continue; + } + } + + // Check the asset fingerprint index. + for (int i = 0; i < ph->asset_fingerprints_size(); i++) { + string value; + const string fp_key = EncodeAssetFingerprintKey(ph->asset_fingerprints(i)); + if (!updates->Get(fp_key, &value)) { + LOG("FSCK: adding missing asset fingerprint key %s for photo %s", fp_key, local_id_str); + updates->Put(fp_key, local_id_str); + changes = true; + } else if (value != local_id_str) { + LOG("FSCK: removing conflicting asset fingerprint %s from photo %s (owned by %s)", + ph->asset_fingerprints(i), local_id_str, value); + ProtoRepeatedFieldRemoveElement(ph->mutable_asset_fingerprints(), i); + // If we modified asset fingerprints, the photo metadata needs to be re-uploaded. + ph->set_update_metadata(true); + save_ph = true; + --i; + continue; + } + } + + if (save_ph) { + LOG("FSCK: rewriting photo %s", *ph); + ph->SaveAndUnlock(updates); + changes = true; + } else { + ph->Unlock(); + } + } + } + + return changes; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PhotoTable.h b/clients/shared/PhotoTable.h new file mode 100644 index 0000000..c6ed673 --- /dev/null +++ b/clients/shared/PhotoTable.h @@ -0,0 +1,233 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_PHOTO_TABLE_H +#define VIEWFINDER_PHOTO_TABLE_H + +#import "ContentTable.h" +#import "Mutex.h" +#import "PhotoMetadata.pb.h" +#import "STLUtils.h" +#import "WallTime.h" + +// The PhotoTable class maintains the mappings: +// -> +// -> +// , -> +// +// PhotoTable is thread-safe and PhotoHandle is thread-safe, but individual +// Photos are not. + +class PhotoTable_Photo : public PhotoMetadata { + friend class PhotoTable; + + public: + virtual void MergeFrom(const PhotoMetadata& m); + // Unimplemented; exists to get the compiler not to complain about hiding the base class's overloaded MergeFrom. + virtual void MergeFrom(const ::google::protobuf::Message&); + + // Return the device/user id for the photo. Returns + // AppState::{device,user}_id if no the photo does not have a device/user id + // set. + int64_t GetDeviceId() const; + int64_t GetUserId() const; + // Returns true if the photo has a valid location. If "location" + // and/or "placemark" are non-NULL, sets their values if available. + // The photo may have a valid location but not a valid placemark. + // In this case, unless it is already busy, the PhotoTable will be + // asked to reverse geocode the location to a placemark. + bool GetLocation(Location* location, Placemark* placemark); + // Returns a formatted location. + string FormatLocation(bool shorten); + // Get the url for the specified name. Returns an empty string if no + // such url exists. + string GetURL(const string& name); + // Get the url for the specified name, returning it only if it has not + // expired. + string GetUnexpiredURL(const string& name, const DBHandle& updates); + // Set the url for the specified name. + void SetURL(const string& name, const string& url, const DBHandle& updates); + // Delete the url for the specified name. + void DeleteURL(const string& name, const DBHandle& updates); + // Returns whether the specified exif timestamp is materially different + // from the photo's timestamp. This is true if the timestamps are not + // equal to the second by any number of integer hour offsets up to 24. + // This accounts for possible timezone ambiguity caused by parsing exif + // datetime string. + bool ShouldUpdateTimestamp(WallTime exif_timestamp); + // Mark the photo as quarantined. + void Quarantine(const string& reason, const DBHandle& updates); + // Returns true iff the photo should be added to an episode. + bool ShouldAddPhotoToEpisode() const; + // Find the set of candidate duplicate photos. + void FindCandidateDuplicates(); + + bool HasAssetUrl() const; + + bool AddAssetKey(const string& asset_key); + bool AddAssetFingerprint(const Slice& asset_fingerprint, bool from_server); + void AddPerceptualFingerprint(const string& term); + void AddPerceptualFingerprint(const ImageFingerprint& fingerprint); + bool RemoveAssetKey(const string& asset_key); + bool RemoveAssetKeyByIndex(int index); + + // Returns true if the photo is available in the library. + bool InLibrary(); + + // Set the server id if it is not already set. Returns true iff the server-id + // was set. + bool MaybeSetServerId(); + + const DBHandle& db() const { return db_; } + + protected: + bool Load(); + void SaveHook(const DBHandle& updates); + void DeleteHook(const DBHandle& updates); + + // Invalidates all activities which have shared this photo and all episodes + // which contain it. + void Invalidate(const DBHandle& updates); + + StringSet GetAssetKeySet() const; + string GetDayTableFields() const; + + int64_t local_id() const { return id().local_id(); } + const string& server_id() const { return id().server_id(); } + + PhotoTable_Photo(AppState* state, const DBHandle& db, int64_t id); + + protected: + AppState* const state_; + DBHandle db_; + + private: + StringSet disk_asset_keys_; + ImageFingerprint disk_perceptual_fingerprint_; + string day_table_fields_; +}; + +class PhotoTable : public ContentTable { + typedef PhotoTable_Photo Photo; + + typedef CallbackSet1 GeocodeCallbackSet; + typedef std::unordered_map< + const Location*, GeocodeCallbackSet*> GeocodeCallbackMap; + typedef Callback GeocodeCallback; + + public: + static const string kPhotoDuplicateQueueKeyPrefix; + + public: + PhotoTable(AppState* state); + ~PhotoTable(); + + void Reset(); + + ContentHandle NewPhoto(const DBHandle& updates) { + return NewContent(updates); + } + ContentHandle LoadPhoto(int64_t id, const DBHandle& db) { + return LoadContent(id, db); + } + ContentHandle LoadPhoto(const string& server_id, const DBHandle& db) { + return LoadContent(server_id, db); + } + ContentHandle LoadPhoto(const PhotoId& id, const DBHandle& db); + + // Loads the photo for the specified asset key. + ContentHandle LoadAssetPhoto(const Slice& asset_key, const DBHandle& db); + + // Returns true if a photo exists with the corresponding url/fingerprint or + // asset key. + bool AssetPhotoExists(const Slice& url, const Slice& fingerprint, + const DBHandle& db); + bool AssetPhotoExists(const Slice& asset_key, const DBHandle& db); + + // The specified assets were not found during a full asset scan. Remove + // references to the asset-urls from PhotoMetadata. + void AssetsNotFound(const StringSet& not_found, const DBHandle& updates); + + // Delete all of the images associated with the photo. + void DeleteAllImages(int64_t photo_id, const DBHandle& updates); + + // Returns true if the photo is available in the library. + bool PhotoInLibrary(int64_t photo_id, const DBHandle& db); + + // Returns true if the two asset photos are equal. Intended to be called from + // asset scans on two photos with the same asset url to identify when iTunes has + // reassigned asset urls to different photos. If the "server" photo predates + // asset fingerprints this method uses heuristics that err on the side of + // assuming asset urls are not reassigned. + static bool IsAssetPhotoEqual( + const PhotoMetadata& local, const PhotoMetadata& server); + + // Reverse geocode the location information for the specified photo, invoking + // the completion callback when done. Returns true if the reverse geocode was + // started or queued and false if reverse geocoding was not required or is + // not available. If false is return, "completion" will not be invoked. + bool ReverseGeocode(int64_t photo_id, GeocodeCallback completion); + + // Resets the error label on the photo handle and re-posts to all + // episodes. + void MaybeUnquarantinePhoto(const ContentHandle& ph, const DBHandle& updates); + void MaybeUnquarantinePhoto(int64_t photo_id); + + // Possibly unquarantine photos. As various fixes are introduced + // which aim to make previously-quarantined photos accessible, an + // internal quarantine version number is incremented. If incremented + // since the last run, this method unquarantines all photos by + // resetting each error label and re-posting the photo to all + // events. Quarantined photos will be re-queued for upload / + // download and if successful, will be re-instated; otherwise, will + // be re-quarantined. + // + // Returns whether photos were unquarantined. + bool MaybeUnquarantinePhotos(ProgressUpdateBlock progress_update); + + private: + friend class PhotoTable_Photo; + + // Start a reverse geocode with the photo manager. Only one reverse + // geocode is kept in-flight by the DayTable at a time. Whether or + // not a day needs a reverse geocode is stored in the DayMetadata. + void MaybeReverseGeocode(int64_t photo_id); + + void MaybeProcessDuplicateQueueLocked(); + + // Looks up the photo id for the specified url/fingerprint or asset_key. If + // require_url_match is true, only returns the photo id if both the url and + // fingerprint portions of the asset key match. + int64_t AssetToDeviceId(const Slice& url, const Slice& fingerprint, + bool require_url_match, const DBHandle& db); + int64_t AssetToDeviceId(const Slice& asset_key, bool require_url_match, + const DBHandle& db); + + // Possibly unquarantines photos and sanity checks metadata. + bool FSCKImpl(int prev_fsck_version, const DBHandle& updates); + + private: + mutable Mutex mu_; + int geocode_in_progress_; + GeocodeCallbackMap geocode_callback_map_; +}; + +typedef PhotoTable::ContentHandle PhotoHandle; + +string EncodeAssetFingerprintKey(const Slice& fingerprint); +string EncodePhotoDuplicateQueueKey(int64_t local_id); +string EncodePerceptualFingerprint(const Slice& term); +bool DecodeAssetFingerprintKey(Slice key, Slice* fingerprint); +bool DecodePhotoDuplicateQueueKey(Slice key, int64_t* local_id); +bool DecodePerceptualFingerprint(Slice fingerprint, string* term); +bool DecodeDeprecatedAssetReverseKey(Slice key, Slice* fingerprint, Slice* url); + +inline bool IsPerceptualFingerprint(const Slice& fingerprint) { + return DecodePerceptualFingerprint(fingerprint, NULL); +} + +#endif // VIEWFINDER_PHOTO_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Placemark.proto b/clients/shared/Placemark.proto new file mode 100644 index 0000000..d608f79 --- /dev/null +++ b/clients/shared/Placemark.proto @@ -0,0 +1,16 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "PlacemarkPB"; + +message Placemark { + optional string iso_country_code = 1; + optional string country = 2; + optional string state = 3; + optional string postal_code = 4; + optional string locality = 5; + optional string sublocality = 6; + optional string thoroughfare = 7; + optional string subthoroughfare = 8; +} diff --git a/clients/shared/PlacemarkHistogram.cc b/clients/shared/PlacemarkHistogram.cc new file mode 100644 index 0000000..a4c1065 --- /dev/null +++ b/clients/shared/PlacemarkHistogram.cc @@ -0,0 +1,410 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import +#import "AppState.h" +#import "Breadcrumb.pb.h" +#import "DB.h" +#import "LazyStaticPtr.h" +#import "LocationUtils.h" +#import "Logging.h" +#import "PhotoMetadata.pb.h" +#import "PlacemarkHistogram.h" +#import "STLUtils.h" +#import "StringUtils.h" + +namespace { + +const string kFormatKey = DBFormat::metadata_key("placemark_histogram_format"); +const string kFormatValue = "4"; +const string kTotalCountKey = DBFormat::metadata_key("placemark_histogram_count"); +// The weightiest placemarks in the histogram up to this percentile +// will be kept in-memory and returned from calls to +// DistanceToTopPlacemark(). +const float kTopPercentile = 0.90; +// A top location must account for this percentage of total photos. +const float kMinPercentile = 0.10; +// No more than this count of top placemarks will be kept in-memory. +const int kTopMaxCount = 5; +// A top location must have at least this many sublocalities, and each +// must have at least this percent of the total count for +// sublocalities to be considered useful. +const int kSublocalityMinCount = 3; +const int kSublocalityMaxCount = 10; +const float kSublocalityMinFraction = 0.05; +const double kHomeVsAwayThresholdDistanceMeters = 100 * 1000; // 100 km + +LazyStaticPtr kSortKeyRE = { "phs/([[:digit:]]+)/(.*)" }; + +// Lowercases provided string and replaces all instances of +// ':' with '-' characters. +string CanonicalizePlaceName(const string& name) { + string lower = ToLowercase(name); + std::replace(lower.begin(), lower.end(), ':', '-'); + return lower; +} + +// Returns the canonical placemark is which country:state:locality. +string GetCanonicalPlacemark(const Placemark& pm) { + return Format("%s:%s:%s", + CanonicalizePlaceName(pm.country()), + CanonicalizePlaceName(pm.state()), + CanonicalizePlaceName(pm.locality())); +} + +// Returns the key for the placemark's histogram entry. +string GetEntryKey(const Placemark& pm) { + const string canon_pm_key = GetCanonicalPlacemark(pm); + return DBFormat::placemark_histogram_key(canon_pm_key); +} + +// Returns the key for the sorted placemark histogram entry. +// Reverse the weight by subtracting from 2^32-1 so the +// sort keys go from highest weight to lowest. +string GetSortedEntryKey(const Placemark& pm, int weight) { + const string canon_pm_key = GetCanonicalPlacemark(pm); + CHECK_LE(weight, 0x7fffffff); + return DBFormat::placemark_histogram_sort_key( + canon_pm_key, 0x7fffffff - weight); +} + +struct SublocalityGreaterThan { + bool operator()(const PlacemarkHistogramEntry::Sublocality* a, + const PlacemarkHistogramEntry::Sublocality* b) const { + if (a->count() != b->count()) { + return a->count() > b->count(); + } + return a->name() < b->name(); + } +}; + +} // namespace + +// The minimum amount of time between placemark histogram inits. +const double PlacemarkHistogram::kMinRefreshIntervalSeconds = 60; + +PlacemarkHistogram::PlacemarkHistogram(AppState* state) + : state_(state), + need_refresh_(true), + last_refresh_(0), + total_count_(state_->db()->Get(kTotalCountKey, 0)) { + const bool format_changed = + (state_->db()->Get(kFormatKey) != kFormatValue); + + if (format_changed) { + DBHandle updates = state_->NewDBTransaction(); + + // Delete all placemark entries and sort keys. + for (DB::PrefixIterator iter(updates, DBFormat::placemark_histogram_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + for (DB::PrefixIterator iter(updates, DBFormat::placemark_histogram_sort_key()); + iter.Valid(); + iter.Next()) { + updates->Delete(iter.key()); + } + // Build histogram with placemarked photos. + total_count_ = 0; + for (DB::PrefixIterator iter(updates, DBFormat::photo_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + PhotoMetadata p; + if (!p.ParseFromArray(value.data(), value.size())) { + LOG("placemark-histogram: unable to parse PhotoMetadata: %s", key); + } else if (p.has_location() && p.has_placemark()) { + UpdateHistogram(p.placemark(), p.location(), 1, updates); + } + } + + updates->Put(kFormatKey, kFormatValue); + updates->Commit(); + LOG("placemark-histogram: built histogram from %d placemarked photo%s", + total_count_, Pluralize(total_count_)); + } +} + +PlacemarkHistogram::~PlacemarkHistogram() { +} + +PlacemarkHistogram::TopPlacemark::TopPlacemark( + const PlacemarkHistogramEntry& e, const int total_count) + : placemark(e.placemark()), + weight(e.count() / total_count), + useful_sublocality(false) { + // Compute location centroid. + if (e.count() > 0) { + centroid.set_latitude(e.location_sum().latitude() / e.count()); + centroid.set_longitude(e.location_sum().longitude() / e.count()); + centroid.set_accuracy(e.location_sum().accuracy() / e.count()); + centroid.set_altitude(e.location_sum().altitude() / e.count()); + } + // Determine whether the sublocalities which are part of this top + // location provide enough differentiation to be useful. + vector counts; + int sublocality_count = 0; + for (int i = 0; i < e.sublocalities_size(); ++i) { + counts.push_back(e.sublocalities(i).count()); + sublocality_count += e.sublocalities(i).count(); + } + // Reverse sort counts from largest to smallest. If a minimum number + // meet the minimal percentile threshold, we conclude that + // sublocalities will be a "useful" addition to this location. + std::sort(counts.begin(), counts.end(), std::greater()); + if (counts.size() >= kSublocalityMaxCount || + (counts.size() >= kSublocalityMinCount && + (float(counts[kSublocalityMinCount - 1]) / sublocality_count) >= kSublocalityMinFraction)) { + useful_sublocality = true; + } +} + +void PlacemarkHistogram::AddPlacemark(const Placemark& placemark, + const Location& location, + const DBHandle& updates) { + // LOG("adding placemark %s, location %s", placemark, location); + UpdateHistogram(placemark, location, 1, updates); +} + +void PlacemarkHistogram::RemovePlacemark(const Placemark& placemark, + const Location& location, + const DBHandle& updates) { + // LOG("removing placemark %s, location %s", placemark, location); + UpdateHistogram(placemark, location, -1, updates); +} + +bool PlacemarkHistogram::DistanceToTopPlacemark(const Location& location, + double* distance, + TopPlacemark* top_placemark) { + const TopPlacemark* closest = FindClosestTopPlacemark(location); + if (!closest) { + return false; + } + if (top_placemark) { + *top_placemark = *closest; + } + *distance = DistanceBetweenLocations(closest->centroid, location); + return true; +} + +bool PlacemarkHistogram::DistanceToLocation( + const Location& location, double* distance, + PlacemarkHistogram::TopPlacemark* top) { + // Find the closest top placemark and format the specified photo's + // placemark relative to it. + if (!DistanceToTopPlacemark(location, distance, top)) { + const Breadcrumb* bc = state_->last_breadcrumb(); + if (!bc) { + return false; + } + *distance = DistanceBetweenLocations(bc->location(), location); + } + return true; +} + +void PlacemarkHistogram::FormatLocation( + const Location& location, const Placemark& placemark, + bool short_location, string* s) { + // Find the closest top placemark and format the specified photo's + // placemark relative to it. If there is no top placemark, use + // the current breadcrumb's placemark. + const Placemark* ref_pm; + bool use_sublocality = false; + double distance; + TopPlacemark top; + if (DistanceToTopPlacemark(location, &distance, &top)) { + ref_pm = &top.placemark; + use_sublocality = (top.useful_sublocality && + distance < kHomeVsAwayThresholdDistanceMeters); + // TODO(peter): Remove the dependency on LocationTracker. + } else if (state_->last_breadcrumb()) { + ref_pm = &state_->last_breadcrumb()->placemark(); + } else { + ref_pm = NULL; + } + + *s = FormatPlacemarkWithReferencePlacemark( + placemark, ref_pm, short_location, + use_sublocality ? PM_SUBLOCALITY : PM_LOCALITY); +} + +void PlacemarkHistogram::FormatLocality( + const Location& location, const Placemark& placemark, string* s) { + double distance; + TopPlacemark top; + if (DistanceToLocation(location, &distance, &top) && + distance < kHomeVsAwayThresholdDistanceMeters && + top.useful_sublocality) { + *s = placemark.sublocality(); + return; + } + *s = placemark.locality(); +} + +void PlacemarkHistogram::MaybeInitTopPlacemarks() { + MutexLock l(&mu_); + if (!need_refresh_ || + (last_refresh_ != 0 && + (state_->WallTime_Now() - last_refresh_) < kMinRefreshIntervalSeconds)) { + return; + } + top_placemarks_.clear(); + need_refresh_ = false; + last_refresh_ = state_->WallTime_Now(); + + // Scan the placemark histogram entries by sort key which + // provides an ordering over placemark entries by count. + int count = 0; + for (DB::PrefixIterator iter(state_->db(), DBFormat::placemark_histogram_sort_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + int weight; + string canon_pm_key; + if (!RE2::FullMatch(key, *kSortKeyRE, &weight, &canon_pm_key)) { + LOG("placemark-histogram: unable to parse placemark hist entry key: %s", key); + continue; + } + PlacemarkHistogramEntry phe; + if (!state_->db()->GetProto(DBFormat::placemark_histogram_key(canon_pm_key), &phe)) { + LOG("placemark-histogram: unable to find placemark %s", canon_pm_key); + continue; + } + if ((count < kTopPercentile * total_count_) && + (phe.count() > kMinPercentile * total_count_) && + (top_placemarks_.size() < kTopMaxCount)) { + count += phe.count(); + top_placemarks_.push_back(TopPlacemark(phe, total_count_)); + VLOG("placemark-histogram: placemark %d, %.1f%%, %.1f%%ile, %d sublocalities: %s", + top_placemarks_.size(), phe.count() * 100.0 / total_count_, + count * 100.0 / total_count_, phe.sublocalities_size(), phe.placemark()); + int sublocality_count = 0; + for (int i = 0; i < std::min(phe.sublocalities_size(), kSublocalityMinCount); ++i) { + sublocality_count += phe.sublocalities(i).count(); + VLOG(" top sublocality %d: %.1f%%, %.1f%%ile: %s", + i, phe.sublocalities(i).count() * 100.0 / phe.count(), + sublocality_count * 100.0 / phe.count(), + phe.sublocalities(i).name()); + } + continue; + } + break; + } +} + +void PlacemarkHistogram::UpdateHistogram(const Placemark& placemark, + const Location& location, + int count, + const DBHandle& updates) { + MutexLock l(&mu_); + PlacemarkHistogramEntry entry; + + if (LookupHistogramEntry(placemark, &entry, updates)) { + // Remove the previous sorted key for the histogram entry. + string prev_sort_key = GetSortedEntryKey(placemark, entry.count()); + updates->Delete(prev_sort_key); + entry.mutable_location_sum()->set_latitude( + entry.location_sum().latitude() + location.latitude() * count); + entry.mutable_location_sum()->set_longitude( + entry.location_sum().longitude() + location.longitude() * count); + entry.mutable_location_sum()->set_accuracy( + entry.location_sum().accuracy() + location.accuracy() * count); + entry.mutable_location_sum()->set_altitude( + entry.location_sum().altitude() + location.altitude() * count); + entry.set_count(entry.count() + count); + + if (placemark.has_sublocality()) { + // TODO(spencer): this almost certainly doesn't matter, but it + // is a linear search and we could easily enough keep the list + // of sublocalities sorted and do a binary search. + PlacemarkHistogramEntry::Sublocality* sublocality = NULL; + for (int i = 0; i < entry.sublocalities_size(); ++i) { + if (entry.sublocalities(i).name() == placemark.sublocality()) { + sublocality = entry.mutable_sublocalities(i); + sublocality->set_count(sublocality->count() + count); + if (sublocality->count() <= 0) { + ProtoRepeatedFieldRemoveElement(entry.mutable_sublocalities(), i); + } + break; + } + } + if (!sublocality && count > 0) { + sublocality = entry.add_sublocalities(); + sublocality->set_name(placemark.sublocality()); + sublocality->set_count(count); + } + // Sort the sublocalities. + std::sort(entry.mutable_sublocalities()->pointer_begin(), + entry.mutable_sublocalities()->pointer_end(), + SublocalityGreaterThan()); + } + } else { + if (count <= 0) { + return; + } + *entry.mutable_placemark() = placemark; + entry.mutable_placemark()->clear_sublocality(); + entry.mutable_placemark()->clear_thoroughfare(); + entry.mutable_placemark()->clear_subthoroughfare(); + *entry.mutable_location_sum() = location; + entry.set_count(count); + + if (placemark.has_sublocality()) { + PlacemarkHistogramEntry::Sublocality* sublocality = entry.add_sublocalities(); + sublocality->set_name(placemark.sublocality()); + sublocality->set_count(sublocality->count() + count); + } + } + + // Write the histogram entry and its sort key. + string entry_key = GetEntryKey(placemark); + if (entry.count() == 0) { + updates->Delete(entry_key); + } else { + updates->PutProto(entry_key, entry); + string sort_key = GetSortedEntryKey(placemark, entry.count()); + updates->Put(sort_key, ""); + } + + // Update the total histogram count. + total_count_ += count; + updates->Put(kTotalCountKey, total_count_); + + need_refresh_ = true; +} + +const PlacemarkHistogram::TopPlacemark* +PlacemarkHistogram::FindClosestTopPlacemark(const Location& location) { + MaybeInitTopPlacemarks(); + + int closest_index = -1; + double closest_distance = std::numeric_limits::max(); + + for (int i = 0; i < top_placemarks_.size(); ++i) { + const TopPlacemark& top_placemark = top_placemarks_[i]; + double distance = DistanceBetweenLocations(top_placemark.centroid, location); + if (closest_index == -1 || distance < closest_distance) { + closest_index = i; + closest_distance = distance; + } + } + return closest_index == -1 ? NULL : &top_placemarks_[closest_index]; +} + +bool PlacemarkHistogram::LookupHistogramEntry(const Placemark& placemark, + PlacemarkHistogramEntry* entry, + const DBHandle& db) { + const string key = GetEntryKey(placemark); + if (!db->GetProto(key, entry)) { + // LOG("placemark-histogram: unable to find placemark %s", key); + return false; + } + + return true; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PlacemarkHistogram.h b/clients/shared/PlacemarkHistogram.h new file mode 100644 index 0000000..3bbefd8 --- /dev/null +++ b/clients/shared/PlacemarkHistogram.h @@ -0,0 +1,101 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_PLACEMARK_HISTOGRAM_H +#define VIEWFINDER_PLACEMARK_HISTOGRAM_H + +#import +#import "DB.h" +#import "Mutex.h" +#import "PlacemarkHistogramEntry.pb.h" +#import "WallTime.h" + +class AppState; +class Location; + +class PlacemarkHistogram { + public: + struct TopPlacemark { + Placemark placemark; + Location centroid; + double weight; + bool useful_sublocality; + + TopPlacemark() : weight(0), useful_sublocality(false) {} + TopPlacemark(const PlacemarkHistogramEntry& e, int total_count); + }; + + public: + PlacemarkHistogram(AppState* state); + ~PlacemarkHistogram(); + + // Adds a placemark to the histogram. + void AddPlacemark(const Placemark& placemark, + const Location& location, + const DBHandle& updates); + + // Removes a placemark from the histogram. + void RemovePlacemark(const Placemark& placemark, + const Location& location, + const DBHandle& updates); + + // Findest the nearest "top" placemark. If there are no top + // placemarks, returns false. Otherwise, returns true, and sets + // *distance to the distance to the top placemark's centroid. If + // top_placemark is not NULL, copies the top placemark's information + // for use by the caller. + bool DistanceToTopPlacemark(const Location& location, + double* distance, + TopPlacemark* top_placemark); + + bool DistanceToLocation(const Location& location, double* distance, + TopPlacemark* top = NULL); + + void FormatLocation(const Location& location, const Placemark& placemark, + bool short_location, string* s); + void FormatLocality(const Location& location, const Placemark& placemark, + string* s); + + private: + // If needs_refresh_ is true, reads the top-weighted placemarks from + // the database and stores them in top_placemarks_. Otherwise, noop. + void MaybeInitTopPlacemarks(); + + // Updates the histogram for the specified placemark/location by + // adjusting its photo count. The total histogram count is also + // updated. + void UpdateHistogram(const Placemark& placemark, + const Location& location, + int count, + const DBHandle& updates); + + // Finds the closest of the top placemarks to the specified + // location. If there are no top placemarks, returns NULL. + const TopPlacemark* FindClosestTopPlacemark(const Location& location); + + // Looks up the histogram in the database by canonicalized + // placemark key. Sets *entry on success and returns true; + // false otherwise. + bool LookupHistogramEntry(const Placemark& placemark, + PlacemarkHistogramEntry* entry, + const DBHandle& db); + + public: + static const double kMinRefreshIntervalSeconds; + + private: + // Vector of location summaries. + typedef vector TopPlacemarkVec; + // Map of placemark keys to updated counts to support batch + // updates efficiently and correctly. + typedef std::unordered_map PlacemarkEntryMap; + + AppState* state_; + mutable Mutex mu_; + TopPlacemarkVec top_placemarks_; + bool need_refresh_; + WallTime last_refresh_; + double total_count_; +}; + +#endif // VIEWFINDER_PLACEMARK_HISTOGRAM_H diff --git a/clients/shared/PlacemarkHistogramEntry.proto b/clients/shared/PlacemarkHistogramEntry.proto new file mode 100644 index 0000000..05f9341 --- /dev/null +++ b/clients/shared/PlacemarkHistogramEntry.proto @@ -0,0 +1,27 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +import "Location.proto"; +import "Placemark.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "PlacemarkHistogramEntryPB"; + +message PlacemarkHistogramEntry { + optional Placemark placemark = 1; + + // The aggregate values for latitude, longitude, accuracy, and altitude. + optional Location location_sum = 2; + + // The total number of photo locations matching this placemark entry. The + // location_sum divided by this count gives the average location. + optional int32 count = 3; + + // A complete set of sublocalities with counts. + message Sublocality { + optional string name = 1; + optional int32 count = 2; + } + + repeated Sublocality sublocalities = 4; +} diff --git a/clients/shared/PlacemarkTable.cc b/clients/shared/PlacemarkTable.cc new file mode 100644 index 0000000..df31769 --- /dev/null +++ b/clients/shared/PlacemarkTable.cc @@ -0,0 +1,119 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "AppState.h" +#import "PlacemarkTable.h" + +namespace { + +const string kPlacemarkKeyPrefix = DBFormat::placemark_key(""); + +const DBRegisterKeyIntrospect kPlacemarkKeyIntrospect( + kPlacemarkKeyPrefix, + [](Slice key) { + Location l; + if (!DecodePlacemarkKey(key, &l)) { + return string(); + } + return string(Format("%.6f,%.6f", l.latitude(), l.longitude())); + }, + [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +void EncodeDouble(string* s, double v) { + int64_t i; + memcpy(&i, &v, sizeof(i)); + OrderedCodeEncodeVarint64(s, i); +} + +double DecodeDouble(Slice* s) { + const int64_t i = OrderedCodeDecodeVarint64(s); + double d; + memcpy(&d, &i, sizeof(i)); + return d; +} + +} // namespace + +string EncodePlacemarkKey(const Location& l) { + string s(kPlacemarkKeyPrefix); + EncodeDouble(&s, l.latitude()); + EncodeDouble(&s, l.longitude()); + // We intentionally do not use Location::accuracy or Location::altitude here. + return s; +} + +bool DecodePlacemarkKey(Slice key, Location* l) { + if (!key.starts_with(kPlacemarkKeyPrefix)) { + return false; + } + key.remove_prefix(kPlacemarkKeyPrefix.size()); + l->set_latitude(DecodeDouble(&key)); + l->set_longitude(DecodeDouble(&key)); + return true; +} + +PlacemarkTable::PlacemarkData::PlacemarkData( + PlacemarkTable* table, const Location& location, const string& key) + : table_(table), + location_(location), + key_(key), + locked_(false), + valid_(false) { +} + +void PlacemarkTable::PlacemarkData::Load(const DBHandle& db) { + valid_ = db->GetProto(EncodePlacemarkKey(location_), this); +} + +void PlacemarkTable::PlacemarkData::SaveAndUnlock(const DBHandle& updates) { + CHECK(locked_); + updates->PutProto(EncodePlacemarkKey(location_), *this); + valid_ = true; + Unlock(); +} + +PlacemarkTable::PlacemarkTable(AppState* state) { +} + +PlacemarkTable::~PlacemarkTable() { +} + +bool PlacemarkTable::IsLocationValid(const Location& location) { + // Disallow almost-certainly problematic 0,0 case. + if (!location.has_latitude() || !location.has_longitude() || + (location.latitude() == 0 && location.longitude() == 0)) { + return false; + } + return true; +} + +bool PlacemarkTable::IsPlacemarkValid(const Placemark& placemark) { + return placemark.has_country() || placemark.has_state() || + placemark.has_locality(); +} + +PlacemarkTable::PlacemarkHandle PlacemarkTable::FindPlacemark( + const Location& location, const DBHandle& db) { + MutexLock l(&mu_); + const string key = EncodePlacemarkKey(location); + PlacemarkData*& p = placemarks_[key]; + if (!p) { + p = new PlacemarkData(this, location, key); + p->Load(db); + } + return PlacemarkHandle(p); +} + +void PlacemarkTable::ReleasePlacemark(PlacemarkData* p) { + MutexLock l(&mu_); + if (p->refcount_.Unref()) { + placemarks_.erase(p->key_); + delete p; + } +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/PlacemarkTable.h b/clients/shared/PlacemarkTable.h new file mode 100644 index 0000000..cdd189e --- /dev/null +++ b/clients/shared/PlacemarkTable.h @@ -0,0 +1,110 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_PLACEMARK_TABLE_H +#define VIEWFINDER_PLACEMARK_TABLE_H + +#import +#import "DB.h" +#import "Location.pb.h" +#import "Mutex.h" +#import "Placemark.pb.h" +#import "ScopedHandle.h" + +class AppState; + +// The PlacemarkTable class maintains the mappings: +// -> +// +// PlacemarkTable and PlacemarkHandle are thread-safe, but individual +// Placemarks are not. +// +// Note that we use exact location comparison. This is pessimistic, but ok +// since we expect to have identical locations when a user takes multiple +// pictures at the same location. +class PlacemarkTable { + public: + class PlacemarkData : public Placemark { + friend class PlacemarkTable; + friend class ScopedHandle; + + public: + void SaveAndUnlock(const DBHandle& updates); + + void Lock() { + mu_.Lock(); + locked_ = true; + } + + void Unlock() { + locked_ = false; + mu_.Unlock(); + } + + bool valid() { return valid_; } + const Location& location() const { return location_; } + + private: + PlacemarkData(PlacemarkTable* table, const Location& location, + const string& key); + + void Load(const DBHandle& db); + + // Increments the content reference count. Only used by PlacemarkHandle. + void Ref() { + refcount_.Ref(); + } + + // Calls content table to decrement reference count and delete the content + // if this is the last remaining reference. Only used by PlacemarkHandle. + void Unref() { + table_->ReleasePlacemark(this); + } + + private: + PlacemarkTable* const table_; + const Location location_; + const string key_; + AtomicRefCount refcount_; + Mutex mu_; + bool locked_; + bool valid_; + }; + + typedef ScopedHandle PlacemarkHandle; + + public: + PlacemarkTable(AppState* state); + ~PlacemarkTable(); + + static bool IsLocationValid(const Location& location); + static bool IsPlacemarkValid(const Placemark& placemark); + + // Retrieve the placemark for the specified location. This will create a new + // placemark (with Placemark::valid() == false) if one doesn't exist. + PlacemarkHandle FindPlacemark(const Location& location, const DBHandle& db); + + // Return a count of the number of referenced placemarks. + int referenced_placemarks() const { + MutexLock l(&mu_); + return placemarks_.size(); + } + + private: + void ReleasePlacemark(PlacemarkData* p); + + private: + mutable Mutex mu_; + std::unordered_map placemarks_; +}; + +typedef PlacemarkTable::PlacemarkHandle PlacemarkHandle; + +string EncodePlacemarkKey(const Location& l); +bool DecodePlacemarkKey(Slice key, Location* l); + +#endif // VIEWFINDER_PLACEMARK_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/QueueMetadata.proto b/clients/shared/QueueMetadata.proto new file mode 100644 index 0000000..5a3696c --- /dev/null +++ b/clients/shared/QueueMetadata.proto @@ -0,0 +1,10 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "QueueMetadataPB"; + +message QueueMetadata { + optional int32 priority = 1; + optional int64 sequence = 2; +} diff --git a/clients/shared/STLUtils.cc b/clients/shared/STLUtils.cc new file mode 100644 index 0000000..874a4dc --- /dev/null +++ b/clients/shared/STLUtils.cc @@ -0,0 +1,7 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "STLUtils.h" + +const ContainerLiteralMaker& ContainerLiteral = *(new ContainerLiteralMaker); +const ContainerLiteralMaker& L = ContainerLiteral; diff --git a/clients/shared/STLUtils.h b/clients/shared/STLUtils.h new file mode 100644 index 0000000..eecb3fb --- /dev/null +++ b/clients/shared/STLUtils.h @@ -0,0 +1,385 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_STL_UTILS_H +#define VIEWFINDER_STL_UTILS_H + +#import +#import +#import +#import +#import +#import +#import +#import "Utils.h" + +typedef std::unordered_set StringSet; +typedef std::unordered_map StringMap; + +struct ContainerLiteralMaker { + template + struct Maker { + Maker() { + } + Maker(A a0) { + args_[0] = a0; + } + Maker(A a0, A a1) { + args_[0] = a0; + args_[1] = a1; + } + Maker(A a0, A a1, A a2) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + } + Maker(A a0, A a1, A a2, A a3) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + } + Maker(A a0, A a1, A a2, A a3, A a4) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + } + Maker(A a0, A a1, A a2, A a3, A a4, A a5) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + args_[5] = a5; + } + Maker(A a0, A a1, A a2, A a3, A a4, A a5, A a6) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + args_[5] = a5; + args_[6] = a6; + } + Maker(A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + args_[5] = a5; + args_[6] = a6; + args_[7] = a7; + } + Maker(A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7, A a8) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + args_[5] = a5; + args_[6] = a6; + args_[7] = a7; + args_[8] = a8; + } + Maker(A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7, A a8, A a9) { + args_[0] = a0; + args_[1] = a1; + args_[2] = a2; + args_[3] = a3; + args_[4] = a4; + args_[5] = a5; + args_[6] = a6; + args_[7] = a7; + args_[8] = a8; + args_[9] = a9; + } + + template + operator std::set() const { + return std::set(&args_[0], &args_[N]); + } + + template + operator std::unordered_set() const { + return std::unordered_set(&args_[0], &args_[N]); + } + + template + operator std::vector() const { + return std::vector(&args_[0], &args_[N]); + } + + A args_[N]; + }; + + template + Maker<1, A> operator()(A a0) const { + return Maker<1, A>(a0); + } + + template + Maker<2, A> operator()(A a0, A a1) const { + return Maker<2, A>(a0, a1); + } + + template + Maker<3, A> operator()(A a0, A a1, A a2) const { + return Maker<3, A>(a0, a1, a2); + } + + template + Maker<4, A> operator()(A a0, A a1, A a2, A a3) const { + return Maker<4, A>(a0, a1, a2, a3); + } + + template + Maker<5, A> operator()(A a0, A a1, A a2, A a3, A a4) const { + return Maker<5, A>(a0, a1, a2, a3, a4); + } + + template + Maker<6, A> operator()(A a0, A a1, A a2, A a3, A a4, A a5) const { + return Maker<6, A>(a0, a1, a2, a3, a4, a5); + } + + template + Maker<7, A> operator()(A a0, A a1, A a2, A a3, A a4, A a5, A a6) const { + return Maker<7, A>(a0, a1, a2, a3, a4, a5, a6); + } + + template + Maker<8, A> operator()( + A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7) const { + return Maker<8, A>(a0, a1, a2, a3, a4, a5, a6, a7); + } + + template + Maker<9, A> operator()( + A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7, A a8) const { + return Maker<9, A>(a0, a1, a2, a3, a4, a5, a6, a7, a8); + } + + template + Maker<10, A> operator()( + A a0, A a1, A a2, A a3, A a4, A a5, A a6, A a7, A a8, A a9) const { + return Maker<10, A>(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9); + } +}; + +extern const ContainerLiteralMaker& ContainerLiteral; +extern const ContainerLiteralMaker& L; + +template +inline const typename T::mapped_type FindOrNull( + const T& container, const K& key) { + typedef typename T::const_iterator const_iterator; + const_iterator iter(container.find(key)); + if (iter == container.end()) { + return NULL; + } + return iter->second; +} + +template +inline typename T::mapped_type FindOrNull( + T* container, const K& key) { + typedef typename T::iterator iterator; + iterator iter(container->find(key)); + if (iter == container->end()) { + return NULL; + } + return iter->second; +} + +template +inline const typename T::mapped_type* FindPtrOrNull( + const T& container, const K& key) { + typedef typename T::const_iterator const_iterator; + const_iterator iter(container.find(key)); + if (iter == container.end()) { + return NULL; + } + return &iter->second; +} + +template +inline typename T::mapped_type* FindPtrOrNull(T* container, const K& key) { + typedef typename T::iterator iterator; + iterator iter(container->find(key)); + if (iter == container->end()) { + return NULL; + } + return &iter->second; +} + +template +inline const typename T::mapped_type FindOrDefault( + const T& container, const K& key, + const typename T::mapped_type& def_value) { + typedef typename T::const_iterator const_iterator; + const_iterator iter(container.find(key)); + if (iter == container.end()) { + return def_value; + } + return iter->second; +} + +template +inline bool ContainsKey(const T& container, const K& key) { + return container.find(key) != container.end(); +} + +#ifdef __OBJC__ + +template +inline void DeletePointer(T* value, std::true_type is_nsobject) { +} + +template +inline void DeletePointer(T* value, std::false_type is_nsobject) { + delete value; +} + +#endif // __OBJC__ + +template +inline void DeleteValue(T& value) { + // Value isn't a pointer. Nothing to do. +} + +template +inline void DeleteValue(T* value) { +#ifdef __OBJC__ + typedef typename std::is_convertible::type is_nsobject; + DeletePointer(value, is_nsobject()); +#else // __OBJC__ + delete value; +#endif // __OBJC__ +} + +template +inline void DeleteValue(std::pair& p) { + DeleteValue(p.first); + DeleteValue(p.second); +} + +template +inline void Clear(T* container) { + if (!container) { + return; + } + for (typename T::iterator iter(container->begin()); + iter != container->end(); + ++iter) { + DeleteValue(*iter); + } + container->clear(); +} + +// Remove an element from a protobuf repeated field. Reorders existing elements (the last element is +// moved to the newly-vacated position) and changes the size of the array. +template +void ProtoRepeatedFieldRemoveElement(google::protobuf::RepeatedPtrField* array, int i) { + array->SwapElements(i, array->size() - 1); + array->RemoveLast(); +} + +template +bool SetsIntersect(const std::set& a, const std::set& b) { + // TODO(ben): it would be more efficient to implement this by hand + // since we don't care about actually accumulating the results. + std::set result; + std::set_intersection( + a.begin(), a.end(), b.begin(), b.end(), + std::insert_iterator >(result, result.end())); + return !result.empty(); +} + +template +int IndexOf(const T& container, const K& key) { + typename T::const_iterator iter = + std::find(container.begin(), container.end(), key); + if (iter == container.end()) { + return -1; + } + return std::distance(container.begin(), iter); +} + +template +inline ostream& Output(ostream& os, I begin, I end) { + os << "<"; + for (I iter(begin); iter != end; ++iter) { + if (iter != begin) { + os << " "; + } + os << *iter; + } + os << ">"; + return os; +} + +template +inline ostream& operator<<( + ostream& os, const ContainerLiteralMaker::Maker& m) { + os << "<"; + for (int i = 0; i < N; ++i) { + if (i > 0) { + os << " "; + } + os << m.args_[i]; + } + os << ">"; + return os; +} + +namespace std { + +template +inline ostream& operator<<(ostream& os, const pair& p) { + os << p.first << ":" << p.second; + return os; +} + +template +inline ostream& operator<<(ostream& os, const vector& v) { + return Output(os, v.begin(), v.end()); +} + +template +inline ostream& operator<<(ostream& os, const set& s) { + return Output(os, s.begin(), s.end()); +} + +template +inline ostream& operator<<(ostream& os, const map& m) { + return Output(os, m.begin(), m.end()); +} + +template +inline ostream& operator<<(ostream& os, const unordered_set& s) { + return Output(os, s.begin(), s.end()); +} + +template +inline ostream& operator<<(ostream& os, const unordered_map& m) { + return Output(os, m.begin(), m.end()); +} + +} // namespace std + +#ifdef __OBJC__ + +// The libc++ hash specialization doesn't work for objective-c +// types. Provide our own specialization that does. +struct HashObjC : public std::unary_function { + size_t operator()(id v) const { + return std::hash()((__bridge void*)v); + } +}; + +#endif // __OBJC__ + +#endif // VIEWFINDER_STL_UTILS_H diff --git a/clients/shared/ScopedHandle.h b/clients/shared/ScopedHandle.h new file mode 100644 index 0000000..0e7d12f --- /dev/null +++ b/clients/shared/ScopedHandle.h @@ -0,0 +1,86 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_SCOPED_HANDLE_H +#define VIEWFINDER_SCOPED_HANDLE_H + +#import + +class AtomicRefCount { + public: + AtomicRefCount() + : val_(0) { + } + AtomicRefCount(const AtomicRefCount& v) + : val_(v.val_.load()) { + } + + // Atomically increments the reference count. + void Ref() { + val_.fetch_add(1); + } + + // Atomically decrements the reference count and returns true iff the + // reference count has reached 0. + bool Unref() { + // std::atomic_int::fetch_sub() returns the previous value before the + // subtraction occurs. + return val_.fetch_sub(1) == 1; + } + + AtomicRefCount& operator=(const AtomicRefCount& v) { + val_.store(v.val_.load()); + return *this; + } + + int32_t val() const { return val_; } + + private: + std::atomic_int val_; +}; + +template +class ScopedHandle { + public: + ScopedHandle(T* ptr = NULL) + : ptr_(ptr) { + if (ptr_) { + ptr_->Ref(); + } + } + ScopedHandle(const ScopedHandle& other) + : ptr_(NULL) { + reset(other); + } + ~ScopedHandle() { + if (ptr_) { + ptr_->Unref(); + } + } + + void reset(const ScopedHandle& other = ScopedHandle()) { + if (ptr_ != other.ptr_) { + if (ptr_) { + ptr_->Unref(); + } + ptr_ = other.ptr_; + if (ptr_) { + ptr_->Ref(); + } + } + } + + T* get() const { return ptr_; } + T* operator->() const { return ptr_; } + T& operator*() const { return *ptr_; } + + ScopedHandle& operator=(const ScopedHandle& other) { + reset(other); + return *this; + } + + private: + T* ptr_; +}; + +#endif // VIEWFINDER_SCOPED_HANDLE_H diff --git a/clients/shared/ScopedPtr.h b/clients/shared/ScopedPtr.h new file mode 100644 index 0000000..ab83624 --- /dev/null +++ b/clients/shared/ScopedPtr.h @@ -0,0 +1,123 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_SCOPED_PTR_H +#define VIEWFINDER_SCOPED_PTR_H + +#import + +template +class ScopedPtr { + public: + ScopedPtr(T* ptr = NULL) + : ptr_(ptr) { + } + ~ScopedPtr() { + reset(NULL); + } + + T* get() const { return ptr_; } + T* operator->() const { return ptr_; } + T& operator*() const { return *ptr_; } + + bool operator==(T* p) const { + return ptr_ == p; + } + bool operator!=(T* p) const { + return ptr_ != p; + } + + void reset(T* new_ptr) { + if (ptr_ != new_ptr) { + enum { type_must_be_complete = sizeof(T) }; + delete ptr_; + ptr_ = new_ptr; + } + } + + void swap(ScopedPtr& other) { + T* tmp = ptr_; + ptr_ = other.ptr_; + other.ptr_ = tmp; + } + + T* release() { + T* released_ptr = ptr_; + ptr_ = NULL; + return released_ptr; + } + + private: + // Disallow evil constructors + ScopedPtr(const ScopedPtr&); + void operator=(const ScopedPtr&); + + private: + T* ptr_; +}; + +template +void swap(ScopedPtr& a, ScopedPtr& b) { + a.swap(b); +} + +template +class ScopedArray { + public: + ScopedArray(T* ptr = NULL) + : ptr_(ptr) { + } + ~ScopedArray() { + reset(NULL); + } + + T* get() const { return ptr_; } + T* operator->() const { return ptr_; } + T& operator*() const { return *ptr_; } + + T& operator[](ptrdiff_t i) const { + return ptr_[i]; + } + + bool operator==(T* p) const { + return ptr_ == p; + } + bool operator!=(T* p) const { + return ptr_ != p; + } + + void reset(T* new_ptr) { + if (ptr_ != new_ptr) { + enum { type_must_be_complete = sizeof(T) }; + delete [] ptr_; + ptr_ = new_ptr; + } + } + + void swap(ScopedArray& other) { + T* tmp = ptr_; + ptr_ = other.ptr_; + other.ptr_ = tmp; + } + + T* release() { + T* released_ptr = ptr_; + ptr_ = NULL; + return released_ptr; + } + + private: + // Disallow evil constructors + ScopedArray(const ScopedArray&); + void operator=(const ScopedArray&); + + private: + T* ptr_; +}; + +template +void swap(ScopedArray& a, ScopedArray& b) { + a.swap(b); +} + +#endif // VIEWFINDER_SCOPED_PTR_H diff --git a/clients/shared/SearchUtils.cc b/clients/shared/SearchUtils.cc new file mode 100644 index 0000000..29d26a7 --- /dev/null +++ b/clients/shared/SearchUtils.cc @@ -0,0 +1,292 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell. + +#import "AppState.h" +#import "CommentTable.h" +#import "ContactManager.h" +#import "DayMetadata.pb.h" +#import "EpisodeTable.h" +#import "FullTextIndex.h" +#import "SearchUtils.h" +#import "Timer.h" +#import "ViewpointTable.h" + +template void PopulateEventSearchResults<>(AppState* state, const DayTable::EventSummaryHandle& events, + vector* results, const Slice& query, std::map* row_map); +template void PopulateEventSearchResults<>(AppState* state, const DayTable::FullEventSummaryHandle& events, + vector* results, const Slice& query, std::map* row_map); + +namespace { + +template +void AddEventRow(const SummaryType& events, int row_index, + vector* results, std::set* row_indexes) { + if (row_index < 0 || ContainsKey(*row_indexes, row_index)) { + return; + } + row_indexes->insert(row_index); + + SummaryRow row; + events->GetSummaryRow(row_index, &row); + row.set_original_row_index(row_index); + results->push_back(row); +} + +template +void AddEpisodeToResults(AppState* state, const SummaryType& events, + int64_t ep_id, vector* results, std::set* row_indexes); + +// For EventSummaryHandle and FullEventSummaryHandle +template +void AddEpisodeToResults(AppState* state, const SummaryType& events, + int64_t ep_id, vector* results, std::set* row_indexes) { + EpisodeHandle eh(state->episode_table()->LoadEpisode(ep_id, state->db())); + if (!eh.get()) { + return; + } + if (eh->InLibrary()) { + // If it's a library episode, add its event to the results. + const int row_index = events->GetEpisodeRowIndex(ep_id); + AddEventRow(events, row_index, results, row_indexes); + } else { + // If it's a conversation episode, see if it has any child episodes in the library (i.e. if it is starred). + vector children; + state->episode_table()->ListEpisodesByParentId(eh->id().local_id(), &children, state->db()); + for (int j = 0; j < children.size(); j++) { + EpisodeHandle child(state->episode_table()->LoadEpisode(children[j], state->db())); + if (!child.get() || !child->InLibrary() || child->CountPhotos() == 0) { + // TODO(ben): if these checks fail, GetEpisodeRowIndex will fail (and log a warning). + // Is it faster to do these checks or just fall through and let GetEpisodeRowIndex fail? + continue; + } + const int row_index = events->GetEpisodeRowIndex(children[j]); + AddEventRow(events, row_index, results, row_indexes); + } + } +} + +void AddViewpointToResults(const DayTable::ConversationSummaryHandle& conversations, int64_t vp_id, + vector* results, std::set* viewpoint_ids) { + if (ContainsKey(*viewpoint_ids, vp_id)) { + return; + } + viewpoint_ids->insert(vp_id); + + const int row_index = conversations->GetViewpointRowIndex(vp_id); + if (row_index < 0) { + return; + } + + SummaryRow row; + conversations->GetSummaryRow(row_index, &row); + row.set_original_row_index(row_index); + results->push_back(row); +} + +} // namespace + +void PopulateEventAutocomplete(AppState* state, SummaryAutocompleteResults* results, const Slice& query) { + if (query.empty()) { + return; + } + + // First get the regular autocomplete results. + FullTextIndex::SuggestionResults sugg_results; + state->episode_table()->episode_index()->GetSuggestions(state->db(), query, &sugg_results); + state->viewpoint_table()->viewpoint_index()->GetSuggestions(state->db(), query, &sugg_results); + for (int i = 0; i < sugg_results.size(); i++) { + results->Add(SummaryTokenInfo( + SummaryTokenInfo::TEXT, + sugg_results[i].second, sugg_results[i].second), + sugg_results[i].first); + } + + sugg_results.clear(); + state->episode_table()->location_index()->GetSuggestions(state->db(), query, &sugg_results); + for (int i = 0; i < sugg_results.size(); i++) { + results->Add(SummaryTokenInfo( + SummaryTokenInfo::LOCATION, + sugg_results[i].second, sugg_results[i].second), + sugg_results[i].first); + } + + vector users; + state->contact_manager()->GetAutocompleteUsers(query, state->episode_table()->episode_index(), &users); + for (auto u : users) { + results->Add(SummaryTokenInfo(SummaryTokenInfo::CONTACT, + u.name, + ContactManager::FormatUserToken(u.user_id)), + u.score); + // TODO(ben): SEARCH: get raw terms + } + + ScopedPtr parsed_query(FullTextQuery::Parse(query, FullTextQuery::PREFIX_MATCH)); + StringSet all_terms; + FullTextQueryTermExtractor extractor(&all_terms); + extractor.VisitNode(*parsed_query); + results->reset_filter_re(FullTextIndex::BuildFilterRE(all_terms)); +} + +void PopulateConversationAutocomplete(AppState* state, SummaryAutocompleteResults* results, const Slice& query) { + if (query.empty()) { + return; + } + StringSet all_terms; + + ScopedPtr parsed_query(FullTextQuery::Parse(query, FullTextQuery::PREFIX_MATCH)); + ViewpointTable::ViewpointSearchResults viewpoint_results; + StringSet seen_titles; + for (ScopedPtr iter( + state->viewpoint_table()->viewpoint_index()->Search(state->db(), *parsed_query)); + iter->Valid(); + iter->Next()) { + const int64_t vp_id = FastParseInt64(iter->doc_id()); + ViewpointHandle vh(state->viewpoint_table()->LoadViewpoint(vp_id, state->db())); + + if (!vh.get() || ContainsKey(seen_titles, vh->title())) { + continue; + } + seen_titles.insert(vh->title()); + + // Our standard autocomplete ranking is by number of matched viewpoints, which would put these + // single-viewpoint rows at the bottom, so give them an artificial boost. + const int score = 10; + results->Add(SummaryTokenInfo( + SummaryTokenInfo::CONVERSATION, + vh->title(), vh->title()), + + score); + iter->GetRawTerms(&all_terms); + } + + FullTextIndex::SuggestionResults sugg_results; + state->comment_table()->comment_index()->GetSuggestions(state->db(), query, &sugg_results); + state->viewpoint_table()->viewpoint_index()->GetSuggestions(state->db(), query, &sugg_results); + state->episode_table()->episode_index()->GetSuggestions(state->db(), query, &sugg_results); + for (int i = 0; i < sugg_results.size(); i++) { + results->Add(SummaryTokenInfo( + SummaryTokenInfo::TEXT, + sugg_results[i].second, sugg_results[i].second), + sugg_results[i].first); + } + + sugg_results.clear(); + state->episode_table()->location_index()->GetSuggestions(state->db(), query, &sugg_results); + for (int i = 0; i < sugg_results.size(); i++) { + results->Add(SummaryTokenInfo( + SummaryTokenInfo::LOCATION, + sugg_results[i].second, sugg_results[i].second), + sugg_results[i].first); + } + + vector users; + state->contact_manager()->GetAutocompleteUsers(query, state->viewpoint_table()->viewpoint_index(), &users); + for (auto u : users) { + results->Add(SummaryTokenInfo(SummaryTokenInfo::CONTACT, + u.name, + ContactManager::FormatUserToken(u.user_id)), + u.score); + // TODO(ben): SEARCH: get raw terms + } + + FullTextQueryTermExtractor extractor(&all_terms); + extractor.VisitNode(*parsed_query); + results->reset_filter_re(FullTextIndex::BuildFilterRE(all_terms)); +} + +template +void PopulateEventSearchResults(AppState* state, const SummaryType& events, + vector* results, const Slice& s, std::map* row_map) { + WallTimer timer; + const string query = ToLowercase(s); + + std::set row_indexes; + EpisodeTable::EpisodeSearchResults episode_results; + state->episode_table()->Search(query, &episode_results); + for (int i = 0; i < episode_results.size(); i++) { + AddEpisodeToResults(state, events, episode_results[i], results, &row_indexes); + } + LOG("event summary: searched episodes in %.0f ms", timer.Milliseconds()); + timer.Restart(); + + ViewpointTable::ViewpointSearchResults viewpoint_results; + state->viewpoint_table()->Search(query, &viewpoint_results); + for (int i = 0; i < viewpoint_results.size(); i++) { + const int64_t vp_id = viewpoint_results[i]; + vector vp_rows; + events->GetViewpointRowIndexes(vp_id, &vp_rows); + for (int j = 0; j < vp_rows.size(); j++) { + AddEventRow(events, vp_rows[j], results, &row_indexes); + } + } + LOG("event summary: searched viewpoints in %.0f ms", timer.Milliseconds()); + + std::sort(results->begin(), results->end(), SummaryRowLessThan()); + int position = 0; + for (int i = 0; i < results->size(); i++) { + (*results)[i].set_position(position); + position += (*results)[i].height(); + if (row_map) { + (*row_map)[(*results)[i].original_row_index()] = i; + } + } +} + +void PopulateConversationSearchResults(AppState* state, const DayTable::ConversationSummaryHandle& conversations, + vector* results, const Slice& s, RowIndexMap* row_map) { + WallTimer timer; + const string query = ToLowercase(s); + std::set viewpoint_ids; + + ViewpointTable::ViewpointSearchResults viewpoint_results; + state->viewpoint_table()->Search(query, &viewpoint_results); + for (int i = 0; i < viewpoint_results.size(); i++) { + AddViewpointToResults(conversations, viewpoint_results[i], results, &viewpoint_ids); + } + LOG("convo summary: searched viewpoints in %.0f ms", timer.Milliseconds()); + + CommentTable::CommentSearchResults comment_results; + state->comment_table()->Search(query, &comment_results); + for (int i = 0; i < comment_results.size(); i++) { + const int64_t vp_id = comment_results[i].first; + AddViewpointToResults(conversations, vp_id, results, &viewpoint_ids); + } + LOG("convo summary: searched comments in %.0f ms", timer.Milliseconds()); + + EpisodeTable::EpisodeSearchResults episode_results; + state->episode_table()->Search(query, &episode_results); + for (int64_t ep_id : episode_results) { + EpisodeHandle eh = state->episode_table()->LoadEpisode(ep_id, state->db()); + if (eh.get()) { + AddViewpointToResults(conversations, eh->viewpoint_id().local_id(), results, &viewpoint_ids); + } + } + + // Find viewpoints followed by users named in the query. + // TODO(ben): Do this with query rewriting instead of a separate pass so we can combine + // user and title terms in one query. Probably needs OR support in the query processor, + // and definitely needs user-to-viewpoint mapping in the full-text index instead of a custom index. + vector contact_results; + state->contact_manager()->Search(query, &contact_results, NULL, + ContactManager::SORT_BY_NAME | ContactManager::VIEWFINDER_USERS_ONLY); + for (int i = 0; i < contact_results.size(); i++) { + vector contact_viewpoints; + state->viewpoint_table()->ListViewpointsForUserId(contact_results[i].user_id(), &contact_viewpoints, + state->db()); + for (int j = 0; j < contact_viewpoints.size(); j++) { + AddViewpointToResults(conversations, contact_viewpoints[j], results, &viewpoint_ids); + } + } + LOG("convo summary: searched viewpoints for %d contacts in %.0fms", contact_results.size(), timer.Milliseconds()); + timer.Restart(); + + std::sort(results->begin(), results->end(), SummaryRowLessThan()); + int position = 0; + for (int i = 0; i < results->size(); i++) { + (*results)[i].set_position(position); + position += (*results)[i].height(); + if (row_map) { + (*row_map)[(*results)[i].original_row_index()] = i; + } + } +} diff --git a/clients/shared/SearchUtils.h b/clients/shared/SearchUtils.h new file mode 100644 index 0000000..ebfdf17 --- /dev/null +++ b/clients/shared/SearchUtils.h @@ -0,0 +1,78 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Ben Darnell. + +#import +#import +#import "DayTable.h" +#import "ScopedPtr.h" +#import "Utils.h" + +class AppState; +class SummaryRow; + +// Maps from unfiltered row indexes to filtered indexes. +typedef std::map RowIndexMap; + +struct SummaryTokenInfo { + public: + enum Type { + TEXT, + CONTACT, + CONVERSATION, + LOCATION, + }; + + SummaryTokenInfo() { + } + + SummaryTokenInfo(Type type, const Slice& display_term, const Slice& query_term) + : type(type), + display_term(display_term.as_string()), + query_term(query_term.as_string()) { + } + + bool operator<(const SummaryTokenInfo& other) const; + + Type type; + string display_term; + string query_term; +}; + + +class SummaryAutocompleteResults { + public: + SummaryAutocompleteResults(); + ~SummaryAutocompleteResults(); + + void Add(const SummaryTokenInfo& token, int score); + + void GetSortedResults(vector >* tokens); + + void reset_filter_re(RE2* re) { filter_re_.reset(re); } + RE2* release_filter_re() { return filter_re_.release(); } + + private: + // This could be an unordered_map if we defined hash and equality functions. + std::map tokens_; + ScopedPtr filter_re_; +}; + +struct SummaryRowLessThan { + bool operator()(const SummaryRow& a, const SummaryRow& b) { + return a.timestamp() > b.timestamp(); + } +}; + +void PopulateEventAutocomplete(AppState* state, SummaryAutocompleteResults* results, const Slice& query); +void PopulateConversationAutocomplete(AppState* state, SummaryAutocompleteResults* results, const Slice& query); + +template +void PopulateEventSearchResults(AppState* state, const SummaryType& events, + vector* results, const Slice& query, RowIndexMap* row_map); +extern template void PopulateEventSearchResults(AppState* state, const DayTable::EventSummaryHandle& events, + vector* results, const Slice& query, RowIndexMap* row_map); +extern template void PopulateEventSearchResults(AppState* state, const DayTable::FullEventSummaryHandle& events, + vector* results, const Slice& query, RowIndexMap* row_map); + +void PopulateConversationSearchResults(AppState* state, const DayTable::ConversationSummaryHandle& conversations, + vector*results, const Slice& query, RowIndexMap* row_map); diff --git a/clients/shared/Server.proto b/clients/shared/Server.proto new file mode 100644 index 0000000..a8d040c --- /dev/null +++ b/clients/shared/Server.proto @@ -0,0 +1,211 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +import "ActivityMetadata.proto"; +import "CommentMetadata.proto"; +import "ContactMetadata.proto"; +import "EpisodeMetadata.proto"; +import "InvalidateMetadata.proto"; +import "PhotoMetadata.proto"; +import "SubscriptionMetadata.proto"; +import "SystemMessage.proto"; +import "UserMetadata.proto"; +import "ViewpointMetadata.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ServerPB"; + +message Headers { + optional int32 version = 1; + optional int32 min_required_version = 2; + optional string op_id = 3; +} + +message PhotoUpdate { + optional PhotoMetadata metadata = 1; + optional string tn_get_url = 2; + optional string tn_put_url = 3; + optional string med_get_url = 4; + optional string med_put_url = 5; + optional string full_get_url = 6; + optional string full_put_url = 7; + optional string orig_get_url = 8; + optional string orig_put_url = 9; +} + +message AccountSettingsMetadata { + optional string email_alerts = 1; + repeated string storage_options = 2; +} + +message AuthResponse { + optional Headers headers = 1; + optional int64 user_id = 2; + optional int64 device_id = 3; + optional int32 token_digits = 4; + optional string cookie = 5; +} + +message ErrorResponse { + enum ErrorId { + OK = 0; + UNKNOWN = 1; + INVALID_JSON_REQUEST = 2; + NO_USER_ACCOUNT = 3; + UPDATE_PWD_NOT_CONFIRMED = 4; + ALREADY_REGISTERED = 5; + NETWORK_UNAVAILABLE = 6; + } + message Error { + optional string method = 1; + optional string text = 2; + optional ErrorId error_id = 3 [default=UNKNOWN]; + } + optional Error error = 1; +} + +message PingResponse { + optional SystemMessage message = 1; +} + +message QueryContactsResponse { + optional Headers headers = 1; + optional string last_key = 2; + repeated ContactMetadata contacts = 3; +} + +message QueryEpisodesResponse { + message Episode { + optional string last_key = 1; + optional EpisodeMetadata metadata = 2; + repeated PhotoUpdate photos = 3; + } + + optional Headers headers = 1; + repeated Episode episodes = 2; +} + +message QueryFollowedResponse { + optional Headers headers = 1; + optional string last_key = 2; + repeated ViewpointMetadata viewpoints = 3; +} + +message QueryNotificationsResponse { + message InlineViewpoint { + optional string viewpoint_id = 1; + optional int64 update_seq = 2; + optional int64 viewed_seq = 3; + } + + message InlineInvalidation { + optional ActivityMetadata activity = 1; + optional InlineViewpoint viewpoint = 2; + optional CommentMetadata comment = 3; + optional UsageMetadata usage = 4; + } + + message Notification { + optional int64 notification_id = 1; + optional string name = 2; + optional int64 sender_id = 3; + optional string op_id = 7; + optional double timestamp = 4; + optional InvalidateMetadata invalidate = 5; + optional InlineInvalidation inline_invalidate = 6; + } + + optional Headers headers = 1; + optional string last_key = 2; + repeated Notification notifications = 3; + optional double retry_after = 4; +} + +message QueryUsersResponse { + optional Headers headers = 1; + repeated group User = 2 { + optional ContactMetadata contact = 3; + repeated ServerSubscriptionMetadata subscriptions = 4; + optional AccountSettingsMetadata account_settings = 5; + optional bool no_password = 7; + } +} + +message QueryViewpointsResponse { + message FollowerMetadata { + optional int64 follower_id = 1; + + optional bool label_removed = 2; + optional bool label_unrevivable = 3; + } + + message Viewpoint { + optional string follower_last_key = 1; + optional string activity_last_key = 2; + optional string episode_last_key = 3; + optional string comment_last_key = 4; + optional ViewpointMetadata metadata = 5; + repeated FollowerMetadata followers = 6; + repeated ActivityMetadata activities = 7; + repeated EpisodeMetadata episodes = 8; + repeated CommentMetadata comments = 9; + } + + optional Headers headers = 1; + repeated Viewpoint viewpoints = 2; +} + +message ResolveContactsResponse { + optional Headers headers = 1; + repeated ContactMetadata contacts = 2; +} + +message RsvpResponse { + message Rsvp { + optional string episode_id = 1; + repeated string labels = 2; + } + + optional Headers headers = 1; + repeated Rsvp rsvps = 2; +} + +message UploadContactsResponse { + optional Headers headers = 1; + repeated string contact_ids = 2; +} + +message UploadEpisodeResponse { + optional Headers headers = 1; + repeated PhotoUpdate photos = 2; +} + +// Operation id & timestamp. +message OpHeaders { + optional int64 op_id = 1; + optional double op_timestamp = 2; +} + +message ServerOperation { + optional OpHeaders headers = 1; + + message RemovePhotos { + repeated ActivityMetadata.Episode episodes = 1; + } + + // The upload_activity operation encapsulates the client-server communication + // to upload a locally generated activity (e.g. share_new, share_existing, + // add_followers, post_comment, etc). + optional int64 upload_activity = 2; + // The update_photo operation encapsulates the somewhat complex client-server + // communication necessary to fully upload a photo's metadata and assets. The + // state for the update_photo operation is kept track of in the PhotoMetadata + // proto. + optional int64 update_photo = 3; + optional RemovePhotos remove_photos = 4; + optional int64 update_viewpoint = 5; + // An array of priorities for future server operations that will be queued + // after this one finishes. For use in accurately tracking how many + // individual pieces of work are contained on the network queue. + repeated int32 stats = 6; +} diff --git a/clients/shared/ServerId.cc b/clients/shared/ServerId.cc new file mode 100644 index 0000000..09a079b --- /dev/null +++ b/clients/shared/ServerId.cc @@ -0,0 +1,138 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#import "DB.h" +#import "Format.h" +#import "Logging.h" +#import "Mutex.h" +#import "ServerId.h" +#import "StringUtils.h" + +namespace { + +// NOTE: these should be kept up to date with the id prefixes used by +// the server. These can be found in backend/db/id_prefix.py. +const char* kActivityPrefix = "a"; +const char* kCommentPrefix = "c"; +const char* kEpisodePrefix = "e"; +const char* kOperationPrefix = "o"; +const char* kPhotoPrefix = "p"; +const char* kViewpointPrefix = "v"; + +bool DecodeId(const char* prefix, const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp, bool reverse_timestamp) { + if (server_id.empty() || + !server_id.starts_with(prefix)) { + return false; + } + + const string decoded = Base64HexDecode(server_id.substr(1)); + Slice s(decoded); + + if (decoded.size() < 4) { + return false; + } + *timestamp = Fixed32Decode(&s); + if (reverse_timestamp) { + *timestamp = (1ULL << 32) - *timestamp - 1; + } + + *device_id = Varint64Decode(&s); + *local_id = Varint64Decode(&s); + return true; +} + +bool DecodeId(const char* prefix, const Slice& server_id, + int64_t* device_id, int64_t* local_id) { + if (server_id.empty() || + !server_id.starts_with(prefix)) { + return false; + } + + const string decoded = Base64HexDecode(server_id.substr(1)); + Slice s(decoded); + + *device_id = Varint64Decode(&s); + *local_id = Varint64Decode(&s); + return true; +} + +string EncodeId(const char* prefix, int64_t device_id, int64_t local_id) { + string encoded; + Varint64Encode(&encoded, device_id); + Varint64Encode(&encoded, local_id); + return Format("%s%s", prefix, Base64HexEncode(encoded, false)); +} + +string EncodeId(const char* prefix, int64_t device_id, int64_t local_id, + WallTime timestamp, bool reverse_timestamp) { + if (timestamp < 0) { + // If timestamp is negative, just use the current time. + timestamp = WallTime_Now(); + } + + string encoded; + if (reverse_timestamp) { + timestamp = (1ULL << 32) - int(timestamp) - 1; + } + Fixed32Encode(&encoded, timestamp); + Varint64Encode(&encoded, device_id); + Varint64Encode(&encoded, local_id); + return Format("%s%s", prefix, Base64HexEncode(encoded, false)); +} + +} // namespace + + +string EncodeActivityId(int64_t device_id, int64_t local_id, WallTime timestamp) { + return EncodeId(kActivityPrefix, device_id, local_id, timestamp, true); +} + +string EncodeCommentId(int64_t device_id, int64_t local_id, WallTime timestamp) { + return EncodeId(kCommentPrefix, device_id, local_id, timestamp, false); +} + +string EncodeEpisodeId(int64_t device_id, int64_t local_id, WallTime timestamp) { + return EncodeId(kEpisodePrefix, device_id, local_id, timestamp, true); +} + +string EncodePhotoId(int64_t device_id, int64_t local_id, WallTime timestamp) { + return EncodeId(kPhotoPrefix, device_id, local_id, timestamp, true); +} + +string EncodeOperationId(int64_t device_id, int64_t local_id) { + return EncodeId(kOperationPrefix, device_id, local_id); +} + +string EncodeViewpointId(int64_t device_id, int64_t local_id) { + return EncodeId(kViewpointPrefix, device_id, local_id); +} + + +bool DecodeActivityId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp) { + return DecodeId(kActivityPrefix, server_id, device_id, local_id, timestamp, true); +} + +bool DecodeCommentId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp) { + return DecodeId(kCommentPrefix, server_id, device_id, local_id, timestamp, false); +} + +bool DecodeEpisodeId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp) { + return DecodeId(kEpisodePrefix, server_id, device_id, local_id, timestamp, true); +} + +bool DecodePhotoId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp) { + return DecodeId(kPhotoPrefix, server_id, device_id, local_id, timestamp, true); +} + +bool DecodeOperationId(const Slice& server_id, int64_t* device_id, int64_t* local_id) { + return DecodeId(kOperationPrefix, server_id, device_id, local_id); +} + +bool DecodeViewpointId(const Slice& server_id, int64_t* device_id, int64_t* local_id) { + return DecodeId(kViewpointPrefix, server_id, device_id, local_id); +} diff --git a/clients/shared/ServerId.h b/clients/shared/ServerId.h new file mode 100644 index 0000000..1e9355e --- /dev/null +++ b/clients/shared/ServerId.h @@ -0,0 +1,46 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Spencer Kimball. + +#ifndef VIEWFINDER_SERVER_ID_H +#define VIEWFINDER_SERVER_ID_H + +#import "WallTime.h" + +// Provides a mapping from the local ids to server ids and back. +// +// Server ids are typically composed of a globally-unique device id, +// established with the server on initial device registration, and +// a device-unique local id created by the allocating device for each +// asset class. +// +// Many server ids, such as photos, episodes, comments and activities, +// also contain a timestamp prefix. For photos, episodes and +// activities, the timestamp is reverse ordered so the most recently +// created assets sort first. Comments, by contrast, sort from least +// recent to most recent. + +string EncodeActivityId(int64_t device_id, int64_t local_id, WallTime timestamp); +string EncodeCommentId(int64_t device_id, int64_t local_id, WallTime timestamp); +string EncodeEpisodeId(int64_t device_id, int64_t local_id, WallTime timestamp); +string EncodePhotoId(int64_t device_id, int64_t local_id, WallTime timestamp); + +string EncodeOperationId(int64_t device_id, int64_t local_id); +string EncodeViewpointId(int64_t device_id, int64_t local_id); + +bool DecodeActivityId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp); +bool DecodeCommentId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp); +bool DecodeEpisodeId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp); +bool DecodePhotoId(const Slice& server_id, int64_t* device_id, + int64_t* local_id, WallTime* timestamp); + +bool DecodeOperationId(const Slice& server_id, int64_t* device_id, int64_t* local_id); +bool DecodeViewpointId(const Slice& server_id, int64_t* device_id, int64_t* local_id); + +#endif // VIEWFINDER_SERVER_ID_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ServerUtils.cc b/clients/shared/ServerUtils.cc new file mode 100644 index 0000000..aa34303 --- /dev/null +++ b/clients/shared/ServerUtils.cc @@ -0,0 +1,1469 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import "ActivityMetadata.pb.h" +#import "AppState.h" +#import "ContactManager.h" +#import "DBFormat.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "Server.pb.h" +#import "ServerUtils.h" +#import "STLUtils.h" +#import "StringUtils.h" + +namespace { + +const string kAssetKeyPrefix = DBFormat::asset_key(""); + +LazyStaticPtr kS3RequestTimeoutErrorRE = { + "RequestTimeout" +}; + +// Add handlers for new photo & episode handlers below. +template +struct LabelHandler { + string label; + bool (T::*has)() const; + bool (T::*getter)() const; + void (T::*setter)(bool v); +}; + +template +class LabelHandlerMap: public std::unordered_map > { +}; + +template +LabelHandlerMap MakeLabelHandlerMap(LabelHandler* handlers, int num_handlers) { + LabelHandlerMap handler_map; + for (int i = 0; i < num_handlers; ++i) { + const LabelHandler& handler = handlers[i]; + handler_map[handler.label] = handler; + } + return handler_map; +} + +typedef PhotoMetadata PM; +LabelHandler kPhotoLabelHandlers[] = { + { "error", &PM::has_label_error, &PM::label_error, &PM::set_label_error }, + { "removed", &PM::has_label_removed, &PM::label_removed, &PM::set_label_removed }, + { "hidden", &PM::has_label_hidden, &PM::label_hidden, &PM::set_label_hidden }, + { "unshared", &PM::has_label_unshared, &PM::label_unshared, &PM::set_label_unshared }, +}; +LabelHandlerMap photo_handler_map = MakeLabelHandlerMap( + kPhotoLabelHandlers, ARRAYSIZE(kPhotoLabelHandlers)); + +typedef EpisodeMetadata EM; +LabelHandler kEpisodeLabelHandlers[] = { +}; +LabelHandlerMap episode_handler_map = MakeLabelHandlerMap( + kEpisodeLabelHandlers, ARRAYSIZE(kEpisodeLabelHandlers)); + +typedef QueryViewpointsResponse::FollowerMetadata FM; +LabelHandler kFollowerLabelHandlers[] = { + { "removed", &FM::has_label_removed, &FM::label_removed, &FM::set_label_removed }, + { "unrevivable", &FM::has_label_unrevivable, &FM::label_unrevivable, &FM::set_label_unrevivable }, +}; +LabelHandlerMap follower_handler_map = MakeLabelHandlerMap( + kFollowerLabelHandlers, ARRAYSIZE(kFollowerLabelHandlers)); + +typedef ViewpointMetadata VM; +LabelHandler kViewpointLabelHandlers[] = { + { "admin", &VM::has_label_admin, &VM::label_admin, &VM::set_label_admin }, + { "contribute", &VM::has_label_contribute, &VM::label_contribute, &VM::set_label_contribute }, + { "hidden", &VM::has_label_hidden, &VM::label_hidden, &VM::set_label_hidden }, + { "muted", &VM::has_label_muted, &VM::label_muted, &VM::set_label_muted }, + { "autosave", &VM::has_label_autosave, &VM::label_autosave, &VM::set_label_autosave }, + { "removed", &VM::has_label_removed, &VM::label_removed, &VM::set_label_removed }, + { "unrevivable", &VM::has_label_unrevivable, &VM::label_unrevivable, &VM::set_label_unrevivable }, +}; +LabelHandlerMap viewpoint_handler_map = MakeLabelHandlerMap( + kViewpointLabelHandlers, ARRAYSIZE(kViewpointLabelHandlers)); + +typedef ContactMetadata CM; +LabelHandler kContactLabelHandlers[] = { + { "removed", &CM::has_label_contact_removed, &CM::label_contact_removed, &CM::set_label_contact_removed }, +}; +LabelHandlerMap contact_handler_map = MakeLabelHandlerMap( + kContactLabelHandlers, ARRAYSIZE(kContactLabelHandlers)); + +LabelHandler kUserLabelHandlers[] = { + { "registered", &CM::has_label_registered, &CM::label_registered, &CM::set_label_registered }, + { "terminated", &CM::has_label_terminated, &CM::label_terminated, &CM::set_label_terminated }, + { "friend", &CM::has_label_friend, &CM::label_friend, &CM::set_label_friend }, + { "system", &CM::has_label_system, &CM::label_system, &CM::set_label_system }, +}; +LabelHandlerMap user_handler_map = MakeLabelHandlerMap( + kUserLabelHandlers, ARRAYSIZE(kUserLabelHandlers)); + +LabelHandler kResolvedContactLabelHandlers[] = { + { "registered", &CM::has_label_registered, &CM::label_registered, &CM::set_label_registered }, +}; +LabelHandlerMap resolved_contact_handler_map = MakeLabelHandlerMap( + kResolvedContactLabelHandlers, ARRAYSIZE(kResolvedContactLabelHandlers)); + +typedef std::unordered_map ErrorIdMap; + +ErrorIdMap MakeErrorIdMap() { + ErrorIdMap m; + m["INVALID_JSON_REQUEST"] = ErrorResponse::INVALID_JSON_REQUEST; + m["NO_USER_ACCOUNT"] = ErrorResponse::NO_USER_ACCOUNT; + m["UPDATE_PWD_NOT_CONFIRMED"] = ErrorResponse::UPDATE_PWD_NOT_CONFIRMED; + m["ALREADY_REGISTERED"] = ErrorResponse::ALREADY_REGISTERED; + return m; +} + +const ErrorIdMap error_id_map = MakeErrorIdMap(); + +typedef std::unordered_map SeverityMap; + +SeverityMap MakeSeverityMap() { + SeverityMap m; + m["SILENT"] = SystemMessage::SILENT; + m["INFO"] = SystemMessage::INFO; + m["ATTENTION"] = SystemMessage::ATTENTION; + m["DISABLE_NETWORK"] = SystemMessage::DISABLE_NETWORK; + return m; +} + +const SeverityMap severity_map = MakeSeverityMap(); + +template +bool MaybeSet( + T* obj, void (T::*setter)(const string&), const JsonRef& value) { + if (value.empty()) { + return false; + } + (obj->*setter)(value.string_value()); + return true; +} + +template +bool MaybeSet( + T* obj, void (T::*setter)(int64_t), const JsonRef& value) { + if (value.empty()) { + return false; + } + (obj->*setter)(value.int64_value()); + return true; +} + +template +bool MaybeSet( + T* obj, void (T::*setter)(int32_t), const JsonRef& value) { + if (value.empty()) { + return false; + } + (obj->*setter)(value.int32_value()); + return true; +} + +template +bool MaybeSet( + T* obj, void (T::*setter)(double), const JsonRef& value) { + if (value.empty()) { + return false; + } + (obj->*setter)(value.double_value()); + return true; +} + +template +void MaybeSet( + T* obj, void (T::*setter)(bool), const JsonRef& value) { + if (value.empty()) { + return; + } + (obj->*setter)(value.bool_value()); +} + +template +void MaybeParseResponseHeaders(T* obj, const JsonRef& d) { + if (d.empty()) { + return; + } + Headers* h = obj->mutable_headers(); + MaybeSet(h, &Headers::set_version, d["version"]); + MaybeSet(h, &Headers::set_min_required_version, d["min_required_version"]); + MaybeSet(h, &Headers::set_op_id, d["op_id"]); +} + +bool ParseAccountSettingsMetadata(AccountSettingsMetadata* a, const JsonRef& d) { + if (d.empty()) { + return false; + } + MaybeSet(a, &AccountSettingsMetadata::set_email_alerts, d["email_alerts"]); + const JsonRef storage_options(d["storage_options"]); + for (int i = 0; i < storage_options.size(); ++i) { + a->add_storage_options(storage_options[i].string_value()); + } + return true; +} + +bool ParseLocation(Location* l, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef Location T; + MaybeSet(l, &T::set_latitude, d["latitude"]); + MaybeSet(l, &T::set_longitude, d["longitude"]); + MaybeSet(l, &T::set_accuracy, d["accuracy"]); + return true; +} + +bool ParsePlacemark(Placemark* p, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef Placemark T; + MaybeSet(p, &T::set_iso_country_code, d["iso_country_code"]); + MaybeSet(p, &T::set_country, d["country"]); + MaybeSet(p, &T::set_state, d["state"]); + MaybeSet(p, &T::set_locality, d["locality"]); + MaybeSet(p, &T::set_sublocality, d["sublocality"]); + MaybeSet(p, &T::set_thoroughfare, d["thoroughfare"]); + MaybeSet(p, &T::set_subthoroughfare, d["subthoroughfare"]); + return true; +} + +bool ParsePhotoId(PhotoId* i, const JsonRef& v) { + if (v.empty()) { + return false; + } + MaybeSet(i, &PhotoId::set_server_id, v); + return true; +} + +bool ParseCommentId(CommentId* i, const JsonRef& v) { + if (v.empty()) { + return false; + } + MaybeSet(i, &CommentId::set_server_id, v); + return true; +} + +bool ParseEpisodeId(EpisodeId* i, const JsonRef& v) { + if (v.empty()) { + return false; + } + MaybeSet(i, &EpisodeId::set_server_id, v); + return true; +} + +bool ParseViewpointId(ViewpointId* i, const JsonRef& v) { + if (v.empty()) { + return false; + } + MaybeSet(i, &ViewpointId::set_server_id, v); + return true; +} + +bool ParseCoverPhoto(CoverPhoto* cp, const JsonRef& d) { + if (d.empty()) { + return false; + } + if (!ParsePhotoId(cp->mutable_photo_id(), d["photo_id"])) { + return false; + } + if (!ParseEpisodeId(cp->mutable_episode_id(), d["episode_id"])) { + return false; + } + // We specifically ignore the potential URLs returned here-- + // they're not necessary for mobile app use as we already + // have photo URLs for cover photo as part of querying the + // episodes to which they belong. + return true; +} + +template +bool HandleLabel(const LabelHandlerMap& handler_map, T* p, const string& label) { + Slice l(label); + if (l.empty()) { + // Skip invalid label. + return false; + } + const LabelHandler* handler = FindPtrOrNull(handler_map, l.ToString()); + if (!handler) { + // Skip unknown labels. + return false; + } + (p->*handler->setter)(true); + return true; +} + +template +void ParseLabels(const LabelHandlerMap& handler_map, T* p, const JsonRef& v) { + if (v.empty()) { + return; + } + + // Set all labels to false before parsing any true values. + for (typename LabelHandlerMap::const_iterator iter = handler_map.begin(); + iter != handler_map.end(); + ++iter) { + (p->*iter->second.setter)(false); + } + + // Set true values from array of set labels. + for (int i = 0; i < v.size(); ++i) { + HandleLabel(handler_map, p, v[i].string_value()); + } +} + +bool ParsePhotoMetadataImage( + PhotoMetadata::Image* i, const JsonRef& md5, const JsonRef& size) { + typedef PhotoMetadata::Image T; + MaybeSet(i, &T::set_md5, md5); + MaybeSet(i, &T::set_size, size); + return i->has_md5() || i->has_size(); +} + +bool ParsePhotoMetadataImages(PhotoMetadata::Images* i, const JsonRef& d) { + if (!ParsePhotoMetadataImage( + i->mutable_tn(), d["tn_md5"], + d["tn_size"])) { + i->clear_tn(); + } + if (!ParsePhotoMetadataImage( + i->mutable_med(), d["med_md5"], + d["med_size"])) { + i->clear_med(); + } + if (!ParsePhotoMetadataImage( + i->mutable_full(), d["full_md5"], + d["full_size"])) { + i->clear_full(); + } + if (!ParsePhotoMetadataImage( + i->mutable_orig(), d["orig_md5"], + d["orig_size"])) { + i->clear_orig(); + } + return i->has_tn() || i->has_med() || i->has_full() || i->has_orig(); +} + +bool ParsePhotoMetadata(PhotoMetadata* p, const JsonRef& d) { + typedef PhotoMetadata T; + if (!ParsePhotoId(p->mutable_id(), d["photo_id"])) { + p->clear_id(); + } + if (!ParsePhotoId(p->mutable_parent_id(), d["parent_id"])) { + p->clear_parent_id(); + } + if (!ParseEpisodeId(p->mutable_episode_id(), d["episode_id"])) { + p->clear_episode_id(); + } + MaybeSet(p, &T::set_user_id, d["user_id"]); + MaybeSet(p, &T::set_sharing_user_id, d["sharing_user_id"]); + MaybeSet(p, &T::set_aspect_ratio, d["aspect_ratio"]); + MaybeSet(p, &T::set_timestamp, d["timestamp"]); + ParseLabels(photo_handler_map, p, d["labels"]); + if (!ParseLocation(p->mutable_location(), d["location"])) { + p->clear_location(); + } + if (!ParsePlacemark(p->mutable_placemark(), d["placemark"])) { + p->clear_placemark(); + } + MaybeSet(p, &T::set_caption, d["caption"]); + MaybeSet(p, &T::set_link, d["link"]); + + if (!ParsePhotoMetadataImages(p->mutable_images(), d)) { + p->clear_images(); + } + + const JsonRef asset_keys(d["asset_keys"]); + for (int i = 0; i < asset_keys.size(); i++) { + Slice fingerprint, url; + const string asset_key = asset_keys[i].string_value(); + if (!DecodeAssetKey(asset_key, &url, &fingerprint)) { + continue; + } + p->add_asset_fingerprints(ToString(fingerprint)); + } + return true; +} + +bool ParsePhotoUpdate( + PhotoUpdate* p, const JsonRef& d) { + if (d.empty()) { + return false; + } + if (!ParsePhotoMetadata(p->mutable_metadata(), d)) { + return false; + } + typedef PhotoUpdate T; + MaybeSet(p, &T::set_tn_get_url, d["tn_get_url"]); + MaybeSet(p, &T::set_tn_put_url, d["tn_put_url"]); + MaybeSet(p, &T::set_med_get_url, d["med_get_url"]); + MaybeSet(p, &T::set_med_put_url, d["med_put_url"]); + MaybeSet(p, &T::set_full_get_url, d["full_get_url"]); + MaybeSet(p, &T::set_full_put_url, d["full_put_url"]); + MaybeSet(p, &T::set_orig_get_url, d["orig_get_url"]); + MaybeSet(p, &T::set_orig_put_url, d["orig_put_url"]); + return true; +} + +bool ParseCommentMetadata(CommentMetadata* v, const JsonRef& d) { + typedef CommentMetadata T; + if (!ParseCommentId(v->mutable_comment_id(), d["comment_id"])) { + v->clear_comment_id(); + } + if (!ParseViewpointId(v->mutable_viewpoint_id(), d["viewpoint_id"])) { + v->clear_viewpoint_id(); + } + MaybeSet(v, &T::set_user_id, d["user_id"]); + MaybeSet(v, &T::set_asset_id, d["asset_id"]); + MaybeSet(v, &T::set_timestamp, d["timestamp"]); + MaybeSet(v, &T::set_message, d["message"]); + return true; +} + +bool ParseEpisodeMetadata(EpisodeMetadata* v, const JsonRef& d) { + typedef EpisodeMetadata T; + if (!ParseEpisodeId(v->mutable_id(), d["episode_id"])) { + v->clear_id(); + } + if (!ParseEpisodeId(v->mutable_parent_id(), d["parent_ep_id"])) { + v->clear_parent_id(); + } + if (!ParseViewpointId(v->mutable_viewpoint_id(), d["viewpoint_id"])) { + v->clear_viewpoint_id(); + } + MaybeSet(v, &T::set_user_id, d["user_id"]); + MaybeSet(v, &T::set_sharing_user_id, d["sharing_user_id"]); + MaybeSet(v, &T::set_timestamp, d["timestamp"]); + MaybeSet(v, &T::set_publish_timestamp, d["publish_timestamp"]); + ParseLabels(episode_handler_map, v, d["labels"]); + MaybeSet(v, &T::set_title, d["title"]); + MaybeSet(v, &T::set_description, d["description"]); + MaybeSet(v, &T::set_name, d["name"]); + return true; +} + +bool ParseFollowerMetadata(QueryViewpointsResponse::FollowerMetadata* fm, const JsonRef& d) { + typedef QueryViewpointsResponse::FollowerMetadata T; + MaybeSet(fm, &T::set_follower_id, d["follower_id"]); + ParseLabels(follower_handler_map, fm, d["labels"]); + return true; +} + +bool ParseViewpointMetadata(ViewpointMetadata* v, const JsonRef& d) { + typedef ViewpointMetadata T; + if (!ParseViewpointId(v->mutable_id(), d["viewpoint_id"])) { + v->clear_id(); + } + MaybeSet(v, &T::set_user_id, d["user_id"]); + MaybeSet(v, &T::set_update_seq, d["update_seq"]); + MaybeSet(v, &T::set_sharing_user_id, d["sharing_user_id"]); + MaybeSet(v, &T::set_title, d["title"]); + MaybeSet(v, &T::set_description, d["description"]); + MaybeSet(v, &T::set_name, d["name"]); + MaybeSet(v, &T::set_type, d["type"]); + MaybeSet(v, &T::set_viewed_seq, d["viewed_seq"]); + if (!ParseCoverPhoto(v->mutable_cover_photo(), d["cover_photo"])) { + v->clear_cover_photo(); + } + ParseLabels(viewpoint_handler_map, v, d["labels"]); + return true; +} + +bool ParseContactMetadata(ContactMetadata* c, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ContactMetadata T; + + MaybeSet(c, &T::set_server_contact_id, d["contact_id"]); + MaybeSet(c, &T::set_contact_source, d["contact_source"]); + MaybeSet(c, &T::set_name, d["name"]); + MaybeSet(c, &T::set_first_name, d["given_name"]); + MaybeSet(c, &T::set_last_name, d["family_name"]); + MaybeSet(c, &T::set_rank, d["rank"]); + const JsonRef identities(d["identities"]); + for (int i = 0; i < identities.size(); i++) { + ContactIdentityMetadata* ci = c->add_identities(); + const JsonRef cid(identities[i]); + MaybeSet(ci, &ContactIdentityMetadata::set_identity, cid["identity"]); + MaybeSet(ci, &ContactIdentityMetadata::set_description, cid["description"]); + MaybeSet(ci, &ContactIdentityMetadata::set_user_id, cid["user_id"]); + DCHECK(ci->has_identity()); + if (ci->has_identity() && !c->has_primary_identity()) { + c->set_primary_identity(ci->identity()); + } + } + ParseLabels(contact_handler_map, c, d["labels"]); + return true; +} + +bool ParseResolvedContactMetadata(ContactMetadata* c, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ContactMetadata CM; + typedef ContactIdentityMetadata CIM; + MaybeSet(c, &CM::set_primary_identity, d["identity"]); + if (c->has_primary_identity()) { + c->add_identities()->set_identity(c->primary_identity()); + } + MaybeSet(c, &CM::set_user_id, d["user_id"]); + MaybeSet(c, &CM::set_name, d["name"]); + // Resolved contacts contain some user data, but they're incomplete, so we need to request a full query later. + c->set_need_query_user(true); + c->set_contact_source(ContactManager::kContactSourceManual); + ParseLabels(resolved_contact_handler_map, c, d["labels"]); + return true; +} + +bool ParseUsageCategoryMetadata(UsageCategoryMetadata* u, const JsonRef& d) { + typedef UsageCategoryMetadata T; + MaybeSet(u, &T::set_num_photos, d["num_photos"]); + MaybeSet(u, &T::set_tn_size, d["tn_size"]); + MaybeSet(u, &T::set_med_size, d["med_size"]); + MaybeSet(u, &T::set_full_size, d["full_size"]); + MaybeSet(u, &T::set_orig_size, d["orig_size"]); + return true; +} + +bool ParseUsageMetadata(UsageMetadata* u, const JsonRef& d) { + const JsonRef& owned_by = d["owned_by"]; + if (!owned_by.empty() && + !ParseUsageCategoryMetadata(u->mutable_owned_by(), owned_by)) { + return false; + } + const JsonRef& shared_by = d["shared_by"]; + if (!shared_by.empty() && + !ParseUsageCategoryMetadata(u->mutable_shared_by(), shared_by)) { + return false; + } + const JsonRef& visible_to = d["visible_to"]; + if (!visible_to.empty() && + !ParseUsageCategoryMetadata(u->mutable_visible_to(), visible_to)) { + return false; + } + + return true; +} + +bool ParseUserMetadata(ContactMetadata* c, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ContactMetadata T; + MaybeSet(c, &T::set_name, d["name"]); + MaybeSet(c, &T::set_user_id, d["user_id"]); + MaybeSet(c, &T::set_first_name, d["given_name"]); + MaybeSet(c, &T::set_last_name, d["family_name"]); + MaybeSet(c, &T::set_nickname, d["nickname"]); + MaybeSet(c, &T::set_email, d["email"]); + MaybeSet(c, &T::set_phone, d["phone"]); + MaybeSet(c, &T::set_merged_with, d["merged_with"]); + ParseLabels(user_handler_map, c, d["labels"]); + + if (!(d.Contains("private") || + (c->label_friend() && c->label_registered()))) { + // The server returns incomplete results for users that are not friends. + // Flag this result as tentative so it won't replace data merged from a contact record + // or prevent requerying this user in the future. + // Prospective users may be "friends" before they are registered. Set the flag in this case + // to allow names from a matching contact to be used until the prospective user registers. + c->set_need_query_user(true); + } + return true; +} + +bool ParseQueryEpisodesEpisode( + QueryEpisodesResponse::Episode* e, const JsonRef& d) { + if (d.empty()) { + return false; + } + + typedef QueryEpisodesResponse::Episode T; + MaybeSet(e, &T::set_last_key, d["last_key"]); + if (!ParseEpisodeMetadata(e->mutable_metadata(), d)) { + e->clear_metadata(); + return false; + } + + { + const JsonRef photos(d["photos"]); + for (int i = 0; i < photos.size(); ++i) { + if (!ParsePhotoUpdate(e->add_photos(), photos[i])) { + return false; + } + } + } + + return true; +} + +template +bool ParseActivityEpisodes(A* activity, const JsonRef& episodes) { + typedef ActivityMetadata::Episode E; + typedef EpisodeId EI; + typedef PhotoId PI; + + for (int i = 0; i < episodes.size(); ++i) { + const JsonRef& ed = episodes[i]; + E* e = activity->add_episodes(); + MaybeSet(e->mutable_episode_id(), &EI::set_server_id, ed["episode_id"]); + const JsonRef photo_ids(ed["photo_ids"]); + for (int j = 0; j < photo_ids.size(); ++j) { + MaybeSet(e->add_photo_ids(), &PI::set_server_id, photo_ids[j]); + } + } + return true; +} + +bool ParseActivityMetadataAddFollowers( + ActivityMetadata::AddFollowers* a, const JsonRef& d) { + if (d.empty()) { + return false; + } + // Add followers includes only user ids, but the activity metadata + // accommodates a full contact metadata. We simply set the user_id. + // The expansive contact metadata is only used for + // locally-constructed activities which are pending server upload. + const JsonRef follower_ids(d["follower_ids"]); + for (int i = 0; i < follower_ids.size(); ++i) { + a->add_contacts()->set_user_id(follower_ids[i].int64_value()); + } + return true; +} + +bool ParseActivityMetadataMergeAccounts( + ActivityMetadata::MergeAccounts* m, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ActivityMetadata::MergeAccounts MA; + MaybeSet(m, &MA::set_target_user_id, d["target_user_id"]); + MaybeSet(m, &MA::set_source_user_id, d["source_user_id"]); + return true; +} + +bool ParseActivityMetadataPostComment( + ActivityMetadata::PostComment* p, const JsonRef& d) { + if (d.empty()) { + return false; + } + if (!ParseCommentId(p->mutable_comment_id(), d["comment_id"])) { + p->clear_comment_id(); + } + return true; +} + +bool ParseActivityMetadataShareExisting( + ActivityMetadata::ShareExisting* s, const JsonRef& d) { + if (d.empty()) { + return false; + } + return ParseActivityEpisodes(s, d["episodes"]); +} + +bool ParseActivityMetadataShareNew( + ActivityMetadata::ShareNew* s, const JsonRef& d) { + if (d.empty()) { + return false; + } + + bool fully_parsed = ParseActivityEpisodes(s, d["episodes"]); + const JsonRef follower_ids(d["follower_ids"]); + for (int i = 0; i < follower_ids.size(); ++i) { + s->add_contacts()->set_user_id(follower_ids[i].int64_value()); + } + return fully_parsed; +} + +bool ParseActivityMetadataUnshare( + ActivityMetadata::Unshare* u, const JsonRef& d) { + if (d.empty()) { + return false; + } + return ParseActivityEpisodes(u, d["episodes"]); +} + +bool ParseActivityMetadataUpdateEpisode( + ActivityMetadata::UpdateEpisode* ue, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef EpisodeId EI; + + bool fully_parsed = true; + for (auto key : d.member_names()) { + if (key == "episode_id") { + MaybeSet(ue->mutable_episode_id(), &EI::set_server_id, d["episode_id"]); + } else { + fully_parsed = false; + } + } + return fully_parsed; +} + +bool ParseActivityMetadataUpdateViewpoint( + ActivityMetadata::UpdateViewpoint* ue, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ViewpointId EI; + + bool fully_parsed = true; + for (auto key : d.member_names()) { + if (key == "viewpoint_id") { + MaybeSet(ue->mutable_viewpoint_id(), &EI::set_server_id, d["viewpoint_id"]); + } else { + fully_parsed = false; + } + } + return fully_parsed; +} + +bool ParseActivityMetadataUploadEpisode( + ActivityMetadata::UploadEpisode* up, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef EpisodeId EI; + typedef PhotoId PI; + + bool fully_parsed = true; + for (auto key : d.member_names()) { + if (key == "episode_id") { + MaybeSet(up->mutable_episode_id(), &EI::set_server_id, d["episode_id"]); + } else if (key == "photo_ids") { + const JsonRef photo_ids(d["photo_ids"]); + for (int i = 0; i < photo_ids.size(); ++i) { + MaybeSet(up->add_photo_ids(), &PI::set_server_id, photo_ids[i]); + } + } else if (key != "viewpoint_id" && + key != "activity_id" && + key != "user_id" && + key != "timestamp") { + fully_parsed = false; + } + } + return fully_parsed; +} + +bool ParseActivityMetadata(ActivityMetadata* a, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef ViewpointId VI; + typedef ActivityId AI; + typedef ActivityMetadata T; + MaybeSet(a->mutable_viewpoint_id(), &VI::set_server_id, d["viewpoint_id"]); + MaybeSet(a->mutable_activity_id(), &AI::set_server_id, d["activity_id"]); + MaybeSet(a, &T::set_user_id, d["user_id"]); + MaybeSet(a, &T::set_timestamp, d["timestamp"]); + MaybeSet(a, &T::set_update_seq, d["update_seq"]); + + const bool add_followers = ParseActivityMetadataAddFollowers( + a->mutable_add_followers(), d["add_followers"]); + if (!add_followers) { + a->clear_add_followers(); + } + + const bool merge_accounts = ParseActivityMetadataMergeAccounts( + a->mutable_merge_accounts(), d["merge_accounts"]); + if (!merge_accounts) { + a->clear_merge_accounts(); + } + + const bool post_comment = ParseActivityMetadataPostComment( + a->mutable_post_comment(), d["post_comment"]); + if (!post_comment) { + a->clear_post_comment(); + } + + const bool share_new = ParseActivityMetadataShareNew( + a->mutable_share_new(), d["share_new"]); + if (!share_new) { + a->clear_share_new(); + } + + const bool share_existing = ParseActivityMetadataShareExisting( + a->mutable_share_existing(), d["share_existing"]); + if (!share_existing) { + a->clear_share_existing(); + } + + const bool unshare = ParseActivityMetadataUnshare( + a->mutable_unshare(), d["unshare"]); + if (!unshare) { + a->clear_unshare(); + } + + const bool update_episode = ParseActivityMetadataUpdateEpisode( + a->mutable_update_episode(), d["update_episode"]); + if (!update_episode) { + a->clear_update_episode(); + } + + const bool update_viewpoint = ParseActivityMetadataUpdateViewpoint( + a->mutable_update_viewpoint(), d["update_viewpoint"]); + if (!update_viewpoint) { + a->clear_update_viewpoint(); + } + + const bool upload_episode = ParseActivityMetadataUploadEpisode( + a->mutable_upload_episode(), d["upload_episode"]); + if (!upload_episode) { + a->clear_upload_episode(); + } + + return (add_followers || post_comment || share_existing || share_new || + unshare || update_episode || update_viewpoint ||upload_episode); +} + +bool ParseEpisodeSelection(EpisodeSelection* e, const JsonRef& ed) { + typedef EpisodeSelection E; + MaybeSet(e, &E::set_episode_id, ed["episode_id"]); + MaybeSet(e, &E::set_get_attributes, ed["get_attributes"]); + MaybeSet(e, &E::set_get_photos, ed["get_photos"]); + MaybeSet(e, &E::set_photo_start_key, ed["photo_start_key"]); + return true; +} + +bool ParseViewpointSelection(ViewpointSelection* v, const JsonRef& vd) { + typedef ViewpointSelection V; + MaybeSet(v, &V::set_viewpoint_id, vd["viewpoint_id"]); + MaybeSet(v, &V::set_get_attributes, vd["get_attributes"]); + MaybeSet(v, &V::set_get_followers, vd["get_followers"]); + MaybeSet(v, &V::set_follower_start_key, vd["follower_start_key"]); + MaybeSet(v, &V::set_get_activities, vd["get_activities"]); + MaybeSet(v, &V::set_activity_start_key, vd["activity_start_key"]); + MaybeSet(v, &V::set_get_episodes, vd["get_episodes"]); + MaybeSet(v, &V::set_episode_start_key, vd["episode_start_key"]); + MaybeSet(v, &V::set_get_comments, vd["get_comments"]); + MaybeSet(v, &V::set_comment_start_key, vd["comment_start_key"]); + return true; +} + +bool ParseUserIdentityMetadata(ContactIdentityMetadata* id, const JsonRef& d) { + if (d.empty()) { + return false; + } + MaybeSet(id, &ContactIdentityMetadata::set_identity, d["identity"]); + // Authority is not used on the client side. + return true; +} + +bool ParseInvalidateMetadata(InvalidateMetadata* inv, const JsonRef& d) { + if (d.empty()) { + return false; + } + + typedef InvalidateMetadata I; + MaybeSet(inv, &I::set_all, d["all"]); + if (inv->all()) { + return true; + } + + const JsonRef& viewpoints = d["viewpoints"]; + for (int i = 0; i < viewpoints.size(); ++i) { + ViewpointSelection vs; + if (ParseViewpointSelection(&vs, viewpoints[i])) { + inv->add_viewpoints()->CopyFrom(vs); + } + } + const JsonRef& episodes = d["episodes"]; + for (int i = 0; i < episodes.size(); ++i) { + EpisodeSelection es; + if (ParseEpisodeSelection(&es, episodes[i])) { + inv->add_episodes()->CopyFrom(es); + } + } + typedef ContactSelection C; + const JsonRef& contacts = d["contacts"]; + if (!contacts.empty()) { + MaybeSet(inv->mutable_contacts(), &C::set_start_key, + contacts["start_key"]); + MaybeSet(inv->mutable_contacts(), &C::set_all, + contacts["all"]); + } + + const JsonRef& users = d["users"]; + if (!users.empty()) { + for (int i = 0; i < users.size(); ++i) { + inv->add_users()->set_user_id(users[i].int64_value()); + } + } + + return true; +} + +bool ParseInlineInvalidation( + QueryNotificationsResponse::InlineInvalidation* ii, const JsonRef& d) { + typedef QueryNotificationsResponse::InlineViewpoint IV; + if (d.empty()) { + return false; + } + bool activity = ParseActivityMetadata( + ii->mutable_activity(), d["activity"]); + if (!activity) { + ii->clear_activity(); + } + const JsonRef& inline_viewpoint = d["viewpoint"]; + if (!inline_viewpoint.empty()) { + IV* iv = ii->mutable_viewpoint(); + MaybeSet(iv, &IV::set_viewpoint_id, inline_viewpoint["viewpoint_id"]); + MaybeSet(iv, &IV::set_update_seq, inline_viewpoint["update_seq"]); + MaybeSet(iv, &IV::set_viewed_seq, inline_viewpoint["viewed_seq"]); + } + + const JsonRef& inline_comment = d["comment"]; + if (!inline_comment.empty()) { + if (!ParseCommentMetadata(ii->mutable_comment(), inline_comment)) { + return false; + } + } + + const JsonRef& inline_user = d["user"]; + if (!inline_user.empty()) { + // For now, usage is the only info in "user". + const JsonRef& usage = inline_user["usage"]; + if (!usage.empty()) { + if (!ParseUsageMetadata(ii->mutable_usage(), usage)) { + return false; + } + } + } + + return true; +} + +bool ParseQueryNotificationsNotification( + QueryNotificationsResponse::Notification* n, const JsonRef& d) { + if (d.empty()) { + return false; + } + typedef QueryNotificationsResponse::InlineInvalidation II; + typedef QueryNotificationsResponse::Notification T; + MaybeSet(n, &T::set_notification_id, d["notification_id"]); + MaybeSet(n, &T::set_name, d["name"]); + MaybeSet(n, &T::set_sender_id, d["sender_id"]); + MaybeSet(n, &T::set_op_id, d["op_id"]); + MaybeSet(n, &T::set_timestamp, d["timestamp"]); + + bool invalidate = ParseInvalidateMetadata(n->mutable_invalidate(), d["invalidate"]); + if (!invalidate) { + n->clear_invalidate(); + } + bool inline_invalidate = + ParseInlineInvalidation(n->mutable_inline_invalidate(), d["inline"]); + if (!inline_invalidate) { + n->clear_inline_invalidate(); + } + return true; +} + +bool ParseQueryViewpointsViewpoint( + QueryViewpointsResponse::Viewpoint* v, const JsonRef& d) { + if (d.empty()) { + return false; + } + + typedef QueryViewpointsResponse::Viewpoint T; + MaybeSet(v, &T::set_follower_last_key, d["follower_last_key"]); + MaybeSet(v, &T::set_activity_last_key, d["activity_last_key"]); + MaybeSet(v, &T::set_episode_last_key, d["episode_last_key"]); + MaybeSet(v, &T::set_comment_last_key, d["comment_last_key"]); + + if (!ParseViewpointMetadata(v->mutable_metadata(), d)) { + v->clear_metadata(); + return false; + } + + { + const JsonRef followers(d["followers"]); + for (int i = 0; i < followers.size(); ++i) { + ParseFollowerMetadata(v->add_followers(), followers[i]); + } + } + + { + const JsonRef activities(d["activities"]); + for (int i = 0; i < activities.size(); ++i) { + ParseActivityMetadata(v->add_activities(), activities[i]); + } + } + + { + const JsonRef episodes(d["episodes"]); + for (int i = 0; i < episodes.size(); ++i) { + ParseEpisodeMetadata(v->add_episodes(), episodes[i]); + } + } + + { + const JsonRef comments(d["comments"]); + for (int i = 0; i < comments.size(); ++i) { + ParseCommentMetadata(v->add_comments(), comments[i]); + } + } + + return true; +} + +ErrorResponse::ErrorId ParseErrorId(const string& s) { + return FindOrDefault(error_id_map, s, ErrorResponse::UNKNOWN); +} + +SystemMessage::Severity ParseSeverity(const string& s) { + return FindOrDefault(severity_map, s, SystemMessage::UNKNOWN); +} + + +} // namespace + +bool IsS3RequestTimeout(int status, const Slice& data) { + return status == 400 && + RE2::PartialMatch(data, *kS3RequestTimeoutErrorRE); +} + +string EncodeAssetKey(const Slice& url, const Slice& fingerprint) { + string s = kAssetKeyPrefix; + url.AppendToString(&s); + if (!fingerprint.empty()) { + s += "#"; + fingerprint.AppendToString(&s); + } + return s; +} + +bool DecodeAssetKey(Slice key, Slice* url, Slice* fingerprint) { + if (!key.starts_with(kAssetKeyPrefix)) { + return false; + } + key.remove_prefix(kAssetKeyPrefix.size()); + const int pos = key.rfind('#'); + if (pos == key.npos) { + // No asset-fingerprint, just the asset-url. + if (url) { + *url = key; + } + } else { + if (url) { + *url = key.substr(0, pos); + } + if (fingerprint) { + *fingerprint = key.substr(pos + 1); + } + } + return true; +} + +bool ParseAuthResponse( + AuthResponse* r, const string& data) { + typedef AuthResponse T; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + MaybeSet(r, &T::set_user_id, d["user_id"]); + MaybeSet(r, &T::set_device_id, d["device_id"]); + MaybeSet(r, &T::set_cookie, d["cookie"]); + MaybeSet(r, &T::set_token_digits, d["token_digits"]); + + // LOG("parse auth:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseErrorResponse( + ErrorResponse* r, const string& data) { + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + typedef ErrorResponse::Error T; + const JsonRef error = d["error"]; + if (error.empty()) { + return false; + } + MaybeSet(r->mutable_error(), &T::set_method, error["method"]); + MaybeSet(r->mutable_error(), &T::set_text, error["message"]); + const JsonRef id = error["id"]; + if (!id.empty()) { + r->mutable_error()->set_error_id(ParseErrorId(id.string_value())); + } + + // LOG("parse error:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParsePingResponse( + PingResponse* p, const string& data) { + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + typedef SystemMessage T; + const JsonRef msg(d["message"]); + if (msg.empty()) { + // Message is optional. + return true; + } + + MaybeSet(p->mutable_message(), &T::set_title, msg["title"]); + MaybeSet(p->mutable_message(), &T::set_body, msg["body"]); + MaybeSet(p->mutable_message(), &T::set_link, msg["link"]); + MaybeSet(p->mutable_message(), &T::set_identifier, msg["identifier"]); + const JsonRef severity = msg["severity"]; + if (!severity.empty()) { + p->mutable_message()->set_severity(ParseSeverity(severity.string_value())); + } + + if (!p->message().has_identifier() || !p->message().has_severity() || + !p->message().has_title() || p->message().severity() == SystemMessage::UNKNOWN) { + return false; + } + + return true; +} + +bool ParseUploadContactsResponse( + UploadContactsResponse* r, const string& data) { + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef contact_ids(d["contact_ids"]); + for (int i = 0; i < contact_ids.size(); i++) { + r->add_contact_ids(contact_ids[i].string_value()); + } + return true; +} + +bool ParseUploadEpisodeResponse( + UploadEpisodeResponse* r, const string& data) { + typedef UploadEpisodeResponse T; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef photos = d["photos"]; + for (int i = 0; i < photos.size(); ++i) { + ParsePhotoUpdate(r->add_photos(), photos[i]); + } + + // LOG("parse upload episode:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryContactsResponse( + QueryContactsResponse* r, ContactSelection* cs, + int limit, const string& data) { + typedef QueryContactsResponse T; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + MaybeSet(r, &T::set_last_key, d["last_key"]); + + const JsonRef contacts(d["contacts"]); + for (int i = 0; i < contacts.size(); ++i) { + ParseContactMetadata(r->add_contacts(), contacts[i]); + } + + if (contacts.size() >= limit) { + cs->set_start_key(r->last_key()); + } + + // LOG("parse query contacts:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryEpisodesResponse( + QueryEpisodesResponse* r, vector* v, + int limit, const string& data) { + typedef QueryEpisodesResponse T; + typedef std::unordered_map< + string, QueryEpisodesResponse::Episode*> EpisodeMap; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + EpisodeMap map; + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef episodes(d["episodes"]); + for (int i = 0; i < episodes.size(); ++i) { + QueryEpisodesResponse::Episode* e = r->add_episodes(); + ParseQueryEpisodesEpisode(e, episodes[i]); + map[e->metadata().id().server_id()] = e; + } + + // Loop over the episode selections and update them to reflect the + // retrieved data. + if (v != NULL) { + for (int i = 0; i < v->size(); ++i) { + EpisodeSelection* s = &(*v)[i]; + QueryEpisodesResponse::Episode* e = FindOrNull(map, s->episode_id()); + if (!e) { + // The episode wasn't returned in the response, perhaps because it was + // deleted or no longer accessible. Validate the entire selection. + continue; + } + if (e->photos_size() >= limit) { + s->set_get_photos(false); + s->set_photo_start_key(e->last_key()); + } + } + } + + // LOG("parse query episodes:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryFollowedResponse( + QueryFollowedResponse* r, const string& data) { + typedef QueryFollowedResponse T; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + MaybeSet(r, &T::set_last_key, d["last_key"]); + + const JsonRef viewpoints(d["viewpoints"]); + for (int i = 0; i < viewpoints.size(); ++i) { + ParseViewpointMetadata(r->add_viewpoints(), viewpoints[i]); + } + + // LOG("parse query followed:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryNotificationsResponse( + QueryNotificationsResponse* r, NotificationSelection* ns, + int limit, const string& data) { + typedef QueryNotificationsResponse T; + + // Get the integer notification id for the first queried key, + // if one was queried. + int64_t exp_first_id = 0; + if (ns->has_last_key() && !ns->last_key().empty()) { + FromString(ns->last_key(), &exp_first_id); + exp_first_id += 1; + } + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + MaybeSet(r, &T::set_last_key, d["last_key"]); + MaybeSet(r, &T::set_retry_after, d["retry_after"]); + + bool nuclear_invalidation = false; + const JsonRef notifications(d["notifications"]); + for (int i = 0; i < notifications.size(); ++i) { + ParseQueryNotificationsNotification( + r->add_notifications(), notifications[i]); + if (r->notifications(i).invalidate().all()) { + nuclear_invalidation = true; + break; + } else if (i == 0) { + // If the query contained a last key, compare the first returned + // notification id to the expected first id. On a gap, we + // trigger nuclear invalidation. + if (exp_first_id != 0 && + r->notifications(i).notification_id() != exp_first_id) { + LOG("notification: notification ids skipped from %d to %d", + exp_first_id, r->notifications(0).notification_id()); + nuclear_invalidation = true; + break; + } else if (r->headers().min_required_version() > + AppState::protocol_version()) { + // Handle the case of a min-required-version too new for client. + ns->set_max_min_required_version(r->headers().min_required_version()); + ns->set_low_water_notification_id(r->notifications(i).notification_id() - 1); + } + } + } + + // If there was a gap in the notification id sequence or if an "all" + // invalidation was encountered, we indicate total invalidation by + // clearing the last_key and setting query_done to false in the + // notification selection. + if (nuclear_invalidation) { + ns->set_last_key(""); + ns->clear_query_done(); + } else { + // Otherwise, set the last key and mark queries for notifications + // "done" if the number of queried notifications is less than the limit. + if (r->has_last_key()) { + ns->set_last_key(r->last_key()); + } + ns->set_query_done(r->notifications_size() < limit); + } + + // LOG("parse query notifications:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryUsersResponse( + QueryUsersResponse* r, const string& data) { + typedef QueryUsersResponse T; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef users(d["users"]); + for (int i = 0; i < users.size(); ++i) { + const JsonRef& user_dict = users[i]; + QueryUsersResponse::User* user_proto = r->add_user(); + ContactMetadata* contact = user_proto->mutable_contact(); + ParseUserMetadata(contact, user_dict); + + const JsonRef private_dict = user_dict["private"]; + if (!private_dict.empty()) { + const JsonRef ids = private_dict["user_identities"]; + if (!ids.empty()) { + for (int j = 0; j < ids.size(); j++) { + const JsonRef id_dict = ids[j]; + ContactIdentityMetadata* id_proto = contact->add_identities(); + ParseUserIdentityMetadata(id_proto, id_dict); + } + } + const JsonRef subs = private_dict["subscriptions"]; + if (!subs.empty()) { + for (int j = 0; j < subs.size(); j++) { + const JsonRef sub_dict = subs[j]; + ServerSubscriptionMetadata* sub_proto = user_proto->add_subscriptions(); + ParseServerSubscriptionMetadata(sub_proto, sub_dict); + } + } + const JsonRef account_settings_dict = private_dict["account_settings"]; + if (!account_settings_dict.empty()) { + ParseAccountSettingsMetadata(user_proto->mutable_account_settings(), account_settings_dict); + } + MaybeSet(user_proto, &QueryUsersResponse::User::set_no_password, private_dict["no_password"]); + } + } + + // LOG("parse query users:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseQueryViewpointsResponse( + QueryViewpointsResponse* r, vector* v, + int limit, const string& data) { + typedef QueryViewpointsResponse T; + typedef std::unordered_map< + string, QueryViewpointsResponse::Viewpoint*> ViewpointMap; + + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + ViewpointMap map; + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef viewpoints(d["viewpoints"]); + for (int i = 0; i < viewpoints.size(); ++i) { + QueryViewpointsResponse::Viewpoint* p = r->add_viewpoints(); + ParseQueryViewpointsViewpoint(p, viewpoints[i]); + map[p->metadata().id().server_id()] = p; + } + + // Loop over the viewpoint selections and update them to reflect the + // retrieved data. + if (v != NULL) { + for (int i = 0; i < v->size(); ++i) { + ViewpointSelection* s = &(*v)[i]; + QueryViewpointsResponse::Viewpoint* p = FindOrNull(map, s->viewpoint_id()); + if (!p) { + // The viewpoint wasn't returned in the response, perhaps because it was + // deleted or no longer accessible. Validate the entire selection. + continue; + } + if (p->followers_size() >= limit) { + s->set_get_followers(false); + s->set_follower_start_key(p->follower_last_key()); + } + if (p->activities_size() >= limit) { + s->set_get_activities(false); + s->set_activity_start_key(p->activity_last_key()); + } + if (p->episodes_size() >= limit) { + s->set_get_episodes(false); + s->set_episode_start_key(p->episode_last_key()); + } + if (p->comments_size() >= limit) { + s->set_get_comments(false); + s->set_comment_start_key(p->comment_last_key()); + } + } + } + + // LOG("parse query viewpoints:\n%s\n%s", d.FormatStyled(), *r); + return true; +} + +bool ParseResolveContactsResponse( + ResolveContactsResponse* r, const string& data) { + JsonValue d; + if (!d.Parse(data)) { + return false; + } + + MaybeParseResponseHeaders(r, d["headers"]); + + const JsonRef contacts(d["contacts"]); + for (int i = 0; i < contacts.size(); i++) { + if (!ParseResolvedContactMetadata(r->add_contacts(), contacts[i])) { + return false; + } + } + + return true; +} + +bool ParseServerSubscriptionMetadata(ServerSubscriptionMetadata* sub, const JsonRef& d) { + if (d.empty()) { + return false; + } + MaybeSet(sub, &ServerSubscriptionMetadata::set_transaction_id, d["transaction_id"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_subscription_id, d["subscription_id"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_timestamp, d["timestamp"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_expiration_ts, d["expiration_ts"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_product_type, d["product_type"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_quantity, d["quantity"]); + MaybeSet(sub, &ServerSubscriptionMetadata::set_payment_type, d["payment_type"]); + return true; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ServerUtils.h b/clients/shared/ServerUtils.h new file mode 100644 index 0000000..e436ba3 --- /dev/null +++ b/clients/shared/ServerUtils.h @@ -0,0 +1,46 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_SERVER_UTILS_H +#define VIEWFINDER_SERVER_UTILS_H + +#import "JsonUtils.h" +#import "Server.pb.h" + +bool IsS3RequestTimeout(int status, const Slice& data); + +string EncodeAssetKey(const Slice& url, const Slice& fingerprint); +bool DecodeAssetKey(Slice key, Slice* url, Slice* fingerprint); + +bool ParseAuthResponse( + AuthResponse* r, const string& data); +bool ParseErrorResponse( + ErrorResponse* r, const string& data); +bool ParsePingResponse( + PingResponse* p, const string& data); +bool ParseQueryContactsResponse( + QueryContactsResponse* r, ContactSelection* cs, + int limit, const string& data); +bool ParseQueryEpisodesResponse( + QueryEpisodesResponse* r, vector* v, + int limit, const string& data); +bool ParseQueryFollowedResponse( + QueryFollowedResponse* r, const string& data); +bool ParseQueryNotificationsResponse( + QueryNotificationsResponse* r, NotificationSelection* ns, + int limit, const string& data); +bool ParseQueryUsersResponse( + QueryUsersResponse* r, const string& data); +bool ParseQueryViewpointsResponse( + QueryViewpointsResponse* r, vector* v, + int limit, const string& data); +bool ParseResolveContactsResponse( + ResolveContactsResponse* r, const string& data); +bool ParseServerSubscriptionMetadata( + ServerSubscriptionMetadata* sub, const JsonRef& dict); +bool ParseUploadContactsResponse( + UploadContactsResponse* r, const string& data); +bool ParseUploadEpisodeResponse( + UploadEpisodeResponse* r, const string& data); + +#endif // VIEWFINDER_SERVER_UTILS_H diff --git a/clients/shared/StringUtils.android.cc b/clients/shared/StringUtils.android.cc new file mode 100644 index 0000000..0166494 --- /dev/null +++ b/clients/shared/StringUtils.android.cc @@ -0,0 +1,30 @@ +// Copryright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault + +#import "StringUtils.h" +#import "StringUtils.android.h" + +std::function localized_case_insensitive_compare; +std::function localized_number_format; +std::function new_uuid; + +int LocalizedCaseInsensitiveCompare(const Slice& a, const Slice& b) { + if (!localized_case_insensitive_compare) { + return a.compare(b); + } + return localized_case_insensitive_compare(a.ToString(), b.ToString()); +} + +string LocalizedNumberFormat(int value) { + if (!localized_number_format) { + return ToString(value); + } + return localized_number_format(value); +} + +string NewUUID() { + if (!new_uuid) { + return ToString(-1); + } + return new_uuid(); +} diff --git a/clients/shared/StringUtils.android.h b/clients/shared/StringUtils.android.h new file mode 100644 index 0000000..43876e8 --- /dev/null +++ b/clients/shared/StringUtils.android.h @@ -0,0 +1,14 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_STRING_UTILS_ANDROID_H +#define VIEWFINDER_STRING_UTILS_ANDROID_H + +#import +#import "Utils.h" + +extern std::function localized_case_insensitive_compare; +extern std::function localized_number_format; +extern std::function new_uuid; + +#endif // VIEWFINDER_STRING_UTILS_ANDROID_H diff --git a/clients/shared/StringUtils.cc b/clients/shared/StringUtils.cc new file mode 100644 index 0000000..9ac9800 --- /dev/null +++ b/clients/shared/StringUtils.cc @@ -0,0 +1,837 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "Mutex.h" +#import "ScopedPtr.h" +#import "StringUtils.h" + +namespace { + +LazyStaticPtr kSqueezeWhitespaceRE = { "[\\pZ\\pC]+" }; + +// Used in ToAsciiLossy. +Mutex transliterator_mutex; +icu::Transliterator* transliterator; + +// Used in WordSplitter. +Mutex word_break_iterator_mutex; +icu::RuleBasedBreakIterator* word_break_iterator; + +// Used in TruncateUTF8. +Mutex char_break_iterator_mutex; +icu::BreakIterator* char_break_iterator; + +const short* MakeBase64DecodingTable(const char* encoding_table) { + short* table = new short[256]; + for (int i = 0; i < 256; i++) { + table[i] = isspace(i) ? -1 : -2; + } + for (int i = 0; i < 64; ++i) { + table[int(encoding_table[i])] = i; + } + return table; +} + +const char kBase64EncodingTable[65] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const short* kBase64DecodingTable = + MakeBase64DecodingTable(kBase64EncodingTable); + +const char kBase64HexEncodingTable[65] = + "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; +const short* kBase64HexDecodingTable = + MakeBase64DecodingTable(kBase64HexEncodingTable); + +string Base64EncodeInternal(const Slice& str, const char* encoding_table, bool padding = true) { + if (str.empty()) { + return string(); + } + + const unsigned char* src = (const unsigned char*)str.data(); + int src_length = str.size(); + string result(((src_length + 2) / 3) * 4, '\0'); + int i = 0; + + // Keep going until we have less than 3 octets + while (src_length > 2) { + DCHECK_LE(i + 4, result.size()); + result[i++] = encoding_table[src[0] >> 2]; + result[i++] = encoding_table[((src[0] & 0x03) << 4) + (src[1] >> 4)]; + result[i++] = encoding_table[((src[1] & 0x0f) << 2) + (src[2] >> 6)]; + result[i++] = encoding_table[src[2] & 0x3f]; + + // We just handled 3 octets of data + src += 3; + src_length -= 3; + } + + // Now deal with the tail end of things + if (src_length != 0) { + DCHECK_LE(i + 4, result.size()); + result[i++] = encoding_table[src[0] >> 2]; + if (src_length > 1) { + result[i++] = encoding_table[((src[0] & 0x03) << 4) + (src[1] >> 4)]; + result[i++] = encoding_table[(src[1] & 0x0f) << 2]; + if (padding) { + result[i++] = '='; + } + } else { + result[i++] = encoding_table[(src[0] & 0x03) << 4]; + if (padding) { + result[i++] = '='; + result[i++] = '='; + } + } + } + + result.resize(i); + return result; +} + +string Base64DecodeInternal(const Slice& str, const short* decoding_table) { + string result(str.size(), '\0'); + int j = 0; + + // Run through the whole string, converting as we go. + for (int i = 0; i < str.size(); ++i) { + int c = str[i]; + if (c == '=') { + break; + } + + c = decoding_table[c]; + if (c == -1) { + // We're at a whitespace, simply skip over + continue; + } else if (c == -2) { + // We're at an invalid character. + return string(); + } + + switch (i % 4) { + case 0: + result[j] = c << 2; + break; + case 1: + result[j++] |= c >> 4; + result[j] = (c & 0x0f) << 4; + break; + case 2: + result[j++] |= c >>2; + result[j] = (c & 0x03) << 6; + break; + case 3: + result[j++] |= c; + break; + } + } + + result.resize(j); + return result; +} + +template +void OrderedCodeEncodeVarintCommon(string *s, T v) { + char buf[sizeof(T) + 1]; + uint8_t* end = reinterpret_cast(&buf[sizeof(T)]); + uint8_t* ptr = end; + + while (v > 0) { + *ptr-- = v & 255; + v >>= 8; + } + const int len = end - ptr; + *ptr = len; + + s->append(reinterpret_cast(ptr), len + 1); +} + +template +T OrderedCodeDecodeVarintCommon(Slice* s) { + if (s->empty()) { + return 0; + } + int len = (*s)[0]; + if ((len + 1) > s->size()) { + return 0; + } + + T v = 0; + const uint8_t* ptr = reinterpret_cast(s->data()) + len; + for (int i = 0; i < len; ++i) { + v |= static_cast(*ptr--) << (i * 8); + } + s->remove_prefix(len + 1); + + return v; +} + +void FixedEncode(string* s, uint64_t v, int bytes, bool big_endian) { + static const int B = 0xff; + int start_length = s->length(); + // This loop encodes the value as little-endian, regardless of + // underlying CPU. + for (int i = 0; i < bytes; i++) { + s->push_back(uint8_t(v & B)); + v >>= 8; + } + // Reverse the result if encoding was requested as big-endian. + if (big_endian) { + reverse(s->begin() + start_length, s->end()); + } +} + +uint64_t FixedDecode(Slice* s, int bytes, bool big_endian) { + CHECK_GE(s->length(), bytes); + uint64_t v = 0; + const uint8_t* ptr = reinterpret_cast(s->data()) + (big_endian ? 0 : bytes - 1); + for (int i = 0; i < bytes; i++) { + v = (v << 8) | *ptr; + ptr += (big_endian ? 1 : -1); + } + s->remove_prefix(bytes); + return v; +} + +// Signed int to string conversion. +template +inline string IntToString(T v) { + char buf[std::numeric_limits::digits10 + 2] = { 0 }; + int i = ARRAYSIZE(buf); + UnsignedT uv = (v < 0) ? static_cast(-v) : + static_cast(v); + do { + buf[--i] = '0' + (uv % 10); + uv /= 10; + } while (uv); + if (v < 0) { + buf[--i] = '-'; + } + return string(&buf[i], ARRAYSIZE(buf) - i); +}; + +// Unsigned int to string conversion. +template +inline string UintToString(T v) { + char buf[std::numeric_limits::digits10 + 2] = { 0 }; + int i = ARRAYSIZE(buf); + do { + buf[--i] = '0' + (v % 10); + v /= 10; + } while (v); + return string(&buf[i], ARRAYSIZE(buf) - i); +} + +struct { + int64_t max_value; + int64_t divisor; + const char* fmt; +} kFormatRanges[] = { + { 1000, 1, "%.0f" }, + { 10000, 1000, "%.1fK" }, + { 1000000, 1000, "%.0fK" }, + { 10000000, 1000000, "%.1fM" }, + { 1000000000, 1000000, "%.0fM" }, + { 10000000000, 1000000000, "%.1fB" }, + { 1000000000000, 1000000000, "%.0fB" }, + { 10000000000000, 1000000000000, "%.1fT" }, + { 1000000000000000, 1000000000000, "%.0fT" }, +}; + +const char kHexChars[] = "0123456789abcdef"; + +} // namespace + +WordSplitter::operator vector() { + vector result; + Split(std::back_insert_iterator >(result)); + return result; +}; + +WordSplitter::operator StringSet() { + StringSet result; + Split(std::insert_iterator(result, result.begin())); + return result; +}; + +template +void WordSplitter::Split(Iterator result) { + MutexLock lock(&word_break_iterator_mutex); + if (!word_break_iterator) { + UErrorCode icu_status = U_ZERO_ERROR; + // In ICU 5.1, the locale passed here doesn't seem to matter for word breaks. It will switch + // between whitespace and dictionary modes automatically based on the characters it encouters. + ScopedPtr break_iter( + icu::BreakIterator::createWordInstance(icu::Locale::getUS(), icu_status)); + if (!break_iter.get() || !U_SUCCESS(icu_status)) { + LOG("failed to create break iterator: %s", icu_status); + return; + } + + // In ICU 5.1, all break iterators are instances of RuleBasedBreakIterator, which adds a method + // we need to distinguish words from runs of punctuation. + if (break_iter->getDynamicClassID() != icu::RuleBasedBreakIterator::getStaticClassID()) { + LOG("got non-rule-based break iterator"); + return; + } + word_break_iterator = (icu::RuleBasedBreakIterator*)break_iter.release(); + } + + + // The UText family of APIs in ICU would let us do this without so much copying. However, + // while BreakIterator supports UText, it can only do basic word breaking in this mode, not the + // dictionary-based CJ segmentation. + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(str_.data(), str_.size())); + word_break_iterator->setText(ustr); + + int pos = word_break_iterator->first(); + while (pos != icu::BreakIterator::DONE) { + int start = pos; + pos = word_break_iterator->next(); + if (word_break_iterator->getRuleStatus() != UBRK_WORD_NONE) { + string substr; + icu::UnicodeString usubstr = ustr.tempSubString(start, pos - start); + usubstr.toUTF8String(substr); + *result++ = substr; + } + } +} + +string Join(const vector& parts, + const string& delim, int begin, int end) { + string res; + end = std::min(parts.size() - 1, end); + for (int i = begin; i <= end; ++i) { + if (i != begin) { + res.append(delim); + } + res.append(parts[i]); + } + return res; +} + +bool Trim(const Slice& str, string* result) { + // UnicodeCharIterators are significantly faster than RE2 when dealing with + // unicode character classes, so use them to remove leading and trailing whitespace. + int first = 0; + int last = str.size(); + for (UnicodeCharIterator iter(str); !iter.Done(); iter.Advance()) { + if (!IsSpaceControlUnicode(iter.Get())) { + first = iter.Position(); + break; + } + } + for (ReverseUnicodeCharIterator iter(str); !iter.Done(); iter.Advance()) { + if (!IsSpaceControlUnicode(iter.Get())) { + break; + } + last = iter.Position(); + } + CHECK_LE(first, last); + *result = str.substr(first, last - first).as_string(); + return first != 0 || last != str.size(); +} + +string Trim(const string& str) { + string result; + Trim(str, &result); + return result; +} + +string NormalizeWhitespace(const Slice& str) { + string trimmed; + Trim(str, &trimmed); + RE2::GlobalReplace(&trimmed, *kSqueezeWhitespaceRE, " "); + return trimmed; +} + +string TruncateUTF8(const Slice &str, int n) { + MutexLock lock(&char_break_iterator_mutex); + if (!char_break_iterator) { + UErrorCode icu_status = U_ZERO_ERROR; + ScopedPtr break_iter( + icu::BreakIterator::createCharacterInstance(icu::Locale::getUS(), icu_status)); + if (!break_iter.get() || !U_SUCCESS(icu_status)) { + LOG("failed to create break iterator: %s", icu_status); + return ""; + } + char_break_iterator = break_iter.release(); + } + + UErrorCode icu_status = U_ZERO_ERROR; + UText* utext = utext_openUTF8(NULL, str.data(), str.size(), &icu_status); + if (!U_SUCCESS(icu_status)) { + return ""; + } + + char_break_iterator->setText(utext, icu_status); + utext_close(utext); + if (!U_SUCCESS(icu_status)) { + return ""; + } + + int pos = char_break_iterator->next(n); + if (pos == icu::BreakIterator::DONE) { + // We hit the end of the string so return the whole thing + return str.as_string(); + } else { + return string(str.data(), pos); + } +} + +Slice RemovePrefix(const Slice& str, const Slice& prefix) { + CHECK(str.starts_with(prefix)); + Slice result(str); + result.remove_prefix(prefix.size()); + return result; +} + +string BinaryToHex(const Slice& b) { + string h(b.size() * 2, '0'); + const uint8_t* p = (const uint8_t*)b.data(); + for (int i = 0; i < b.size(); ++i) { + const int c = p[i]; + h[2 * i] = kHexChars[c >> 4]; + h[2 * i + 1] = kHexChars[c & 0xf]; + } + return h; +} + +string Base64Encode(const Slice& str) { + return Base64EncodeInternal(str, kBase64EncodingTable); +} + +string Base64HexEncode(const Slice& str, bool padding) { + return Base64EncodeInternal(str, kBase64HexEncodingTable, padding); +} + +string Base64Decode(const Slice& str) { + return Base64DecodeInternal(str, kBase64DecodingTable); +} + +string Base64HexDecode(const Slice& str) { + return Base64DecodeInternal(str, kBase64HexDecodingTable); +} + +string ToLowercase(const Slice& str) { + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8( + icu::StringPiece(str.data(), str.size())); + // TODO(peter): Should we be specifying the locale here? + ustr.toLower(); + string utf8; + ustr.toUTF8String(utf8); + return utf8; +} + +string ToUppercase(const Slice& str) { + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8( + icu::StringPiece(str.data(), str.size())); + // TODO(peter): Should we be specifying the locale here? + ustr.toUpper(); + string utf8; + ustr.toUTF8String(utf8); + return utf8; +} + +string ToAsciiLossy(const Slice& str) { + // If the string is already ascii-only we can exit early. + bool ascii_only = true; + for (int i = 0; i < str.size(); i++) { + if (str[i] & 0x80) { + ascii_only = false; + break; + } + } + if (ascii_only) { + return str.as_string(); + } + + MutexLock lock(&transliterator_mutex); + if (!transliterator) { + UErrorCode icu_status = U_ZERO_ERROR; + // ICU's Any-Latin conversion is best-effort; non-letter characters are left as-is, so + // add an extra step at the end to strip out any non-ascii characters that remain. + transliterator = icu::Transliterator::createInstance( + "Any-Latin; Latin-ASCII; [^\\u0020-\\u007f] Any-Remove", + UTRANS_FORWARD, icu_status); + if (!transliterator || icu_status != U_ZERO_ERROR) { + return ""; + } + } + + icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(str.data(), str.size())); + transliterator->transliterate(ustr); + string ascii; + ustr.toUTF8String(ascii); + return ascii; +} + +void Fixed32Encode(string* s, uint32_t v, bool big_endian) { + FixedEncode(s, v, 4, big_endian); +} + +uint32_t Fixed32Decode(Slice* s, bool big_endian) { + return FixedDecode(s, 4, big_endian); +} + +void Fixed64Encode(string* s, uint64_t v, bool big_endian) { + FixedEncode(s, v, 8, big_endian); +} + +uint64_t Fixed64Decode(Slice* s, bool big_endian) { + return FixedDecode(s, 8, big_endian); +} + +void Varint64Encode(string* s, uint64_t v) { + static const int B = 128; + unsigned char buf[10]; + unsigned char* ptr = buf; + while (v >= B) { + *(ptr++) = (v & (B-1)) | B; + v >>= 7; + } + *(ptr++) = static_cast(v); + s->append(reinterpret_cast(buf), ptr - buf); +} + +uint64_t Varint64Decode(Slice* s) { + static const int B = 128; + const uint8_t* b = reinterpret_cast(s->begin()); + const uint8_t* e = reinterpret_cast(s->end()); + const uint8_t* p = b; + uint64_t result = 0; + for (uint32_t shift = 0; shift <= 63 && p < e; shift += 7) { + uint64_t byte = *p++; + if (byte & B) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + break; + } + } + s->remove_prefix(p - b); + return result; +} + +void OrderedCodeEncodeVarint32(string* s, uint32_t v) { + OrderedCodeEncodeVarintCommon(s, v); +} + +void OrderedCodeEncodeVarint32Decreasing(string* s, uint32_t v) { + return OrderedCodeEncodeVarint32(s, ~v); +} + +void OrderedCodeEncodeVarint64(string* s, uint64_t v) { + OrderedCodeEncodeVarintCommon(s, v); +} + +void OrderedCodeEncodeVarint64Decreasing(string* s, uint64_t v) { + OrderedCodeEncodeVarint64(s, ~v); +} + +uint32_t OrderedCodeDecodeVarint32(Slice* s) { + return OrderedCodeDecodeVarintCommon(s); +} + +uint32_t OrderedCodeDecodeVarint32Decreasing(Slice* s) { + return ~OrderedCodeDecodeVarint32(s); +} + +uint64_t OrderedCodeDecodeVarint64(Slice* s) { + return OrderedCodeDecodeVarintCommon(s); +} + +uint64_t OrderedCodeDecodeVarint64Decreasing(Slice* s) { + return ~OrderedCodeDecodeVarint64(s); +} + +void OrderedCodeEncodeInt64Pair(string* s, int64_t a, int64_t b) { + OrderedCodeEncodeVarint64(s, a); + if (b != 0) { + // Only encode "b" if it is non-zero. This is a slight space optimization, + // but more importantly it allows us to encode only "a" so that we can + // easily create the string prefix that finds all pairs that begin with + // "a". + OrderedCodeEncodeVarint64(s, b); + } +} + +void OrderedCodeDecodeInt64Pair(Slice* s, int64_t *a, int64_t *b) { + *a = OrderedCodeDecodeVarint64(s); + *b = OrderedCodeDecodeVarint64(s); +} + +string FormatCount(int64_t count) { + for (int i = 0; i < ARRAYSIZE(kFormatRanges); ++i) { + if (fabs(count) < kFormatRanges[i].max_value) { + return Format(kFormatRanges[i].fmt, (double(count) / kFormatRanges[i].divisor)); + } + } + return ToString(count); +} + +int64_t FastParseInt64(const Slice& s) { + int64_t x = 0; + if (s[0] == '-') { + for (int i = 1; i < s.size(); i++) { + x = x * 10 - (s[i] - '0'); + } + } else { + for (int i = 0; i < s.size(); i++) { + x = x * 10 + (s[i] - '0'); + } + } + return x; +} + +string Int32ToString(int32_t v) { + return IntToString(v); +} + +string Int64ToString(int64_t v) { + return IntToString(v); +} + +string Uint32ToString(uint32_t v) { + return UintToString(v); +} + +string Uint64ToString(uint64_t v) { + return UintToString(v); +} + +string GzipEncode(const Slice& str) { + if (str.size() == 0) { + return NULL; + } + + z_stream zlib; + memset(&zlib, 0, sizeof(zlib)); + zlib.zalloc = Z_NULL; + zlib.zfree = Z_NULL; + zlib.opaque = Z_NULL; + zlib.total_out = 0; + zlib.next_in = (Bytef*)str.data(); + zlib.avail_in = str.size(); + + int error = deflateInit2( + &zlib, Z_DEFAULT_COMPRESSION, Z_DEFLATED, (15+16), 8, Z_DEFAULT_STRATEGY); + if (error != Z_OK) { + switch (error) { + case Z_STREAM_ERROR: + LOG("deflateInit2() error: Invalid parameter passed in to function: %s", + zlib.msg); + break; + case Z_MEM_ERROR: + LOG("deflateInit2() error: Insufficient memory: %s", + zlib.msg); + break; + case Z_VERSION_ERROR: + LOG("deflateInit2() error: The version of zlib.h and the version " + "of the library linked do not match: %s", zlib.msg); + break; + default: + LOG("deflateInit2() error: Unknown error code %d: %s", + error, zlib.msg); + break; + } + return NULL; + } + + string compressed; + do { + compressed.resize(std::max(1024, compressed.size() * 2)); + zlib.next_out = (Bytef*)compressed.data() + zlib.total_out; + zlib.avail_out = compressed.size() - zlib.total_out; + + error = deflate(&zlib, Z_FINISH); + } while (error == Z_OK); + + if (error != Z_STREAM_END) { + switch (error) { + case Z_ERRNO: + LOG("deflate() error: Error occured while reading file: %s", + zlib.msg); + break; + case Z_STREAM_ERROR: + LOG("deflate() error: : %s", zlib.msg); + LOG("deflate() error: The stream state was inconsistent " + "(e.g. next_in or next_out was NULL): %s", zlib.msg); + break; + case Z_DATA_ERROR: + LOG("deflate() error: The deflate data was invalid or " + "incomplete: %s", zlib.msg); + break; + case Z_MEM_ERROR: + LOG("deflate() error: Memory could not be allocated for " + "processing: %s", zlib.msg); + break; + case Z_BUF_ERROR: + LOG("deflate() error: Ran out of output buffer for writing " + "compressed bytes: %s", zlib.msg); + break; + case Z_VERSION_ERROR: + LOG("deflate() error: The version of zlib.h and the version " + "of the library linked do not match: %s", zlib.msg); + break; + default: + LOG("deflate() error: Unknown error code %d: %s", error, zlib.msg); + break; + } + return NULL; + } + + compressed.resize(zlib.total_out); + deflateEnd(&zlib); + return compressed; +} + +string GzipDecode(const Slice& str) { + if (str.size() == 0) { + return NULL; + } + + z_stream zlib; + memset(&zlib, 0, sizeof(zlib)); + zlib.zalloc = Z_NULL; + zlib.zfree = Z_NULL; + zlib.opaque = Z_NULL; + zlib.total_out = 0; + zlib.next_in = (Bytef*)str.data(); + zlib.avail_in = str.size(); + + int error = inflateInit2(&zlib, (15+16)); + if (error != Z_OK) { + switch (error) { + case Z_STREAM_ERROR: + LOG("inflateInit2() error: Invalid parameter passed in to function: %s", + zlib.msg); + break; + case Z_MEM_ERROR: + LOG("inflateInit2() error: Insufficient memory: %s", + zlib.msg); + break; + case Z_VERSION_ERROR: + LOG("inflateInit2() error: The version of zlib.h and the version " + "of the library linked do not match: %s", zlib.msg); + break; + default: + LOG("inflateInit2() error: Unknown error code %d: %s", + error, zlib.msg); + break; + } + return NULL; + } + + string decompressed; + do { + decompressed.resize(std::max(1024, decompressed.size() * 2)); + zlib.next_out = (Bytef*)decompressed.data() + zlib.total_out; + zlib.avail_out = decompressed.size() - zlib.total_out; + + error = inflate(&zlib, Z_FINISH); + } while (error == Z_OK); + + if (error != Z_STREAM_END) { + switch (error) { + case Z_ERRNO: + LOG("inflate() error: Error occured while reading file: %s", + zlib.msg); + break; + case Z_STREAM_ERROR: + LOG("inflate() error: : %s", zlib.msg); + LOG("inflate() error: The stream state was inconsistent " + "(e.g. next_in or next_out was NULL): %s", zlib.msg); + break; + case Z_DATA_ERROR: + LOG("inflate() error: The inflate data was invalid or " + "incomplete: %s", zlib.msg); + break; + case Z_MEM_ERROR: + LOG("inflate() error: Memory could not be allocated for " + "processing: %s", zlib.msg); + break; + case Z_BUF_ERROR: + LOG("inflate() error: Ran out of output buffer for writing " + "compressed bytes: %s", zlib.msg); + break; + case Z_VERSION_ERROR: + LOG("inflate() error: The version of zlib.h and the version " + "of the library linked do not match: %s", zlib.msg); + break; + default: + LOG("inflate() error: Unknown error code %d: %s", error, zlib.msg); + break; + } + return NULL; + } + + decompressed.resize(zlib.total_out); + inflateEnd(&zlib); + return decompressed; +} + +UnicodeCharIterator::UnicodeCharIterator(const Slice& s) + : error_(false) { + UErrorCode icu_status = U_ZERO_ERROR; + utext_ = utext_openUTF8(NULL, s.data(), s.size(), &icu_status); + if (!U_SUCCESS(icu_status)) { + error_ = true; + } + next_ = utext_next32From(utext_, 0); +} + +UnicodeCharIterator::~UnicodeCharIterator() { + utext_close(utext_); +} + +ReverseUnicodeCharIterator::ReverseUnicodeCharIterator(const Slice& s) + : error_(false) { + UErrorCode icu_status = U_ZERO_ERROR; + utext_ = utext_openUTF8(NULL, s.data(), s.size(), &icu_status); + if (!U_SUCCESS(icu_status)) { + error_ = true; + } + next_ = utext_previous32From(utext_, utext_nativeLength(utext_)); +} + +ReverseUnicodeCharIterator::~ReverseUnicodeCharIterator() { + utext_close(utext_); +} + +bool IsAlphaUnicode(UChar32 c) { + // "GC" here means "general category" + return U_GET_GC_MASK(c) & U_GC_L_MASK; +} + +bool IsAlphaNumUnicode(UChar32 c) { + // "GC" here means "general category" + const int32_t kAlphaNumMask = U_GC_L_MASK | U_GC_N_MASK; + return U_GET_GC_MASK(c) & kAlphaNumMask; +} + +bool IsSpaceControlUnicode(UChar32 c) { + // "GC" here means "general category" + const int32_t kSpaceControlMask = U_GC_Z_MASK | U_GC_C_MASK; + return U_GET_GC_MASK(c) & kSpaceControlMask; +} + +// NOTE(peter): This method is missing from re2. +namespace re2 { +void StringPiece::AppendToString(string* target) const { + target->append(ptr_, length_); +} +} // namespace re2 + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/StringUtils.h b/clients/shared/StringUtils.h new file mode 100644 index 0000000..2807f0c --- /dev/null +++ b/clients/shared/StringUtils.h @@ -0,0 +1,429 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_STRING_UTILS_H +#define VIEWFINDER_STRING_UTILS_H + +#import +#import +#import +#import +#import "STLUtils.h" +#import "Utils.h" + +class Splitter { + public: + Splitter(const string& str, const string& delim, bool allow_empty) + : str_(str), + delim_(delim), + allow_empty_(allow_empty) { + } + + operator vector() { + vector result; + Split(std::back_insert_iterator >(result)); + return result; + } + + private: + template + void Split(Iterator result) { + if (str_.empty()) { + return; + } + for (string::size_type begin_index, end_index = 0; ;) { + begin_index = str_.find_first_not_of(delim_, end_index); + if (begin_index == string::npos) { + MaybeOutputEmpty(result, (end_index == 0) + str_.size() - end_index); + return; + } + MaybeOutputEmpty(result, (end_index == 0) + begin_index - end_index - 1); + end_index = str_.find_first_of(delim_, begin_index); + if (end_index == string::npos) { + end_index = str_.size(); + } + *result++ = str_.substr(begin_index, (end_index - begin_index)); + } + } + + template + void MaybeOutputEmpty(Iterator result, int count) { + if (!allow_empty_) { + return; + } + for (int i = 0; i < count; ++i) { + *result++ = string(); + } + } + + private: + const string& str_; + const string& delim_; + const bool allow_empty_; +}; + +inline Splitter Split(const string& str, const string& delim) { + return Splitter(str, delim, false); +} + +inline Splitter SplitAllowEmpty(const string& str, const string& delim) { + return Splitter(str, delim, true); +} + +class WordSplitter { + public: + WordSplitter(const Slice& str) + : str_(str) { + } + + operator vector(); + operator StringSet(); + + private: + template + void Split(Iterator result); + + const Slice& str_; +}; + +// Split the given string into words in an i18n-aware way. +inline WordSplitter SplitWords(const Slice& str) { + return WordSplitter(str); +} + +template +string Join(Iter begin, Iter end, const string& delim) { + string res; + for (int i = 0; begin != end; ++i, ++begin) { + if (i != 0) { + res.append(delim); + } + res.append(*begin); + } + return res; +} + +string Join(const vector& parts, const string& delim, + int begin = 0, int end = std::numeric_limits::max()); + +// Trim is UTF8 aware and will handle all unicode whitespace (Z, C) +// classes. Returns true if any characters were trimmed from the +// input string. The trimmed result is stored in *result. +bool Trim(const Slice& str, string* result); + +string Trim(const string& str); + +// Removes all leading and trailing whitespace, and replaces all +// repeated internal whitespace (plus any control characters) with a +// single space. Note that this method applies to the unicode +// whitespace (Z) and control character (C) classes +string NormalizeWhitespace(const Slice& str); + +// Performs a localized case insensitive string comparison, returning -1 if a < +// b, 0 if a == b and +1 if a > b. +int LocalizedCaseInsensitiveCompare(const Slice& a, const Slice& b); + +// Performs a localized formatting of the specifying number. +string LocalizedNumberFormat(int value); + +// Returns the first N characters from str. This function knows about utf8 character boundaries +// as well as combining characters and surrogates. +string TruncateUTF8(const Slice& str, int n); + +// Asserts that "str" begins with "prefix" and returns the remaining portion of "str". +Slice RemovePrefix(const Slice& str, const Slice& prefix); + +string BinaryToHex(const Slice& s); + +// Returns a new "formatted" 128-bit UUID. On iOS the string is formatted as: +// +// aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee +// +// Each non-hyphen character is a hexadecimal value. +string NewUUID(); + +string Base64Encode(const Slice& str); +string Base64HexEncode(const Slice& str, bool padding = true); +string Base64Decode(const Slice& str); +string Base64HexDecode(const Slice& str); + +string ToLowercase(const Slice& str); +string ToUppercase(const Slice& str); +// Converts an 8-bit ASCII string (possibly lossily) into a 7-bit +// ASCII string. If the specified string is not convertible into +// 8-bit ASCII, an empty string is returned. +string ToAsciiLossy(const Slice& str); + +void Fixed32Encode(string* s, uint32_t v, bool big_endian = true); +uint32_t Fixed32Decode(Slice* s, bool big_endian = true); +void Fixed64Encode(string* s, uint64_t v, bool big_endian = true); +uint64_t Fixed64Decode(Slice* s, bool big_endian = true); + +void Varint64Encode(string* s, uint64_t v); +uint64_t Varint64Decode(Slice* s); + +void OrderedCodeEncodeVarint32(string* s, uint32_t v); +void OrderedCodeEncodeVarint32Decreasing(string* s, uint32_t v); +void OrderedCodeEncodeVarint64(string* s, uint64_t v); +void OrderedCodeEncodeVarint64Decreasing(string* s, uint64_t v); +uint32_t OrderedCodeDecodeVarint32(Slice* s); +uint32_t OrderedCodeDecodeVarint32Decreasing(Slice* s); +uint64_t OrderedCodeDecodeVarint64(Slice* s); +uint64_t OrderedCodeDecodeVarint64Decreasing(Slice* s); + +void OrderedCodeEncodeInt64Pair(string* s, int64_t a, int64_t b); +void OrderedCodeDecodeInt64Pair(Slice* s, int64_t *a, int64_t *b); + +// Formats a count using abbreviations for thousands. +// 1-999: "1"-"999" +// 1,000-9,999: "1.0K"-"9.9K" +// 10,000-999,000: "10K"-"999K" +// 1,000,000-9,999,999: "1.0M"-"9.9M" +// etc. including "B", and "T". +string FormatCount(int64_t count); + +inline Slice Pluralize( + int n, const char* singular = "", const char* plural = "s") { + return (n == 1) ? singular : plural; +} + +// Parses a decimal integer from the given string, which must contain only digits. +// Does no bounds checking; behavior on overflow is undefined. +// This is much faster than strtoll on iOS because strtoll uses division in its bounds +// checking, and 64-bit integer division is implemented in software on iOS devices. +int64_t FastParseInt64(const Slice& s); + +template +struct ToStringImpl { + static string Convert(const T& t) { + std::ostringstream s; + s.precision(std::numeric_limits::digits10 + 1); + s << t; + return s.str(); + } +}; + +template <> +struct ToStringImpl { + inline static string Convert(const Slice& s) { + return s.ToString(); + } +}; + +template <> +struct ToStringImpl { + inline static string Convert(const string& s) { + return s; + } +}; + +// Faster int to string conversion. +string Int32ToString(int32_t v); +string Int64ToString(int64_t v); +string Uint32ToString(uint32_t v); +string Uint64ToString(uint64_t v); + +string GzipEncode(const Slice& str); +string GzipDecode(const Slice& str); + +template <> +struct ToStringImpl { + inline static string Convert(int32_t v) { + return Int32ToString(v); + } +}; + +template <> +struct ToStringImpl { + inline static string Convert(int64_t v) { + return Int64ToString(v); + } +}; + +template <> +struct ToStringImpl { + inline static string Convert(uint32_t v) { + return Uint32ToString(v); + } +}; + +template <> +struct ToStringImpl { + inline static string Convert(uint64_t v) { + return Uint64ToString(v); + } +}; + +template +inline string ToString(const T &t) { + return ToStringImpl::Convert(t); +} + +template +inline void FromString(const string &str, T *val) { + std::istringstream s(str); + s >> *val; +} + +inline void FromString(const string& str, string* val) { + *val = str; +} + +template +inline T FromString(const string &str, T val = T()) { + std::istringstream s(str); + s >> val; + return val; +} + +template +inline T FromString(const Slice &str, T val = T()) { + return FromString(str.as_string(), val); +} + +inline Slice ToSlice(const Slice& s) { + return s; +} + +#ifdef __OBJC__ + +#import +#import +#import + +inline NSString* NewNSString(const Slice& s) { + return [[NSString alloc] initWithBytes:s.data() + length:s.size() + encoding:NSUTF8StringEncoding]; +} + +inline NSString* NewNSString(const char* s) { + return NewNSString(Slice(s)); +} + +inline NSString* NewNSString(const string& s) { + return NewNSString(Slice(s)); +} + +inline NSData* NewNSData(const Slice& s) { + return [[NSData alloc] initWithBytes:s.data() + length:s.size()]; +} + +inline NSData* NewNSData(const string& s) { + return NewNSData(Slice(s)); +} + +inline NSURL* NewNSURL(const Slice& s) { + return [[NSURL alloc] initWithString:NewNSString(s)]; +} + +inline NSURL* NewNSURL(const string& s) { + return NewNSURL(Slice(s)); +} + +inline string ToString(NSString* s) { + if (s) { + return [s UTF8String]; + } + return ""; +} + +inline string ToString(NSData* d) { + if (d) { + return string((const char*)d.bytes, d.length); + } + return ""; +} + +inline string ToString(NSObject* o) { + return ToString([o description]); +} + +inline string ToString(NSURL *u) { + return ToString([u description]); +} + +inline Slice ToSlice(NSString* s) { + if (s) { + return [s UTF8String]; + } + return Slice(); +} + +inline Slice ToSlice(NSData* d) { + if (d) { + return Slice((const char*)d.bytes, d.length); + } + return Slice(); +} + +#endif // __OBJC__ + +// Iterates over unicode characters in the given (UTF-8 encoded) string. +// Note that this class uses real 32-bit characters instead of the UTF-16 codepoints used +// in both NSString and icu::UnicodeString. +class UnicodeCharIterator { + public: + explicit UnicodeCharIterator(const Slice& s); + + ~UnicodeCharIterator(); + + bool error() const { return error_; } + + UChar32 Get() const { return next_; } + + bool Done() const { return next_ < 0; } + + void Advance() { + next_ = utext_next32(utext_); + } + + // Returns the position of the first byte of the current character. + int64_t Position() const { + return utext_getPreviousNativeIndex(utext_); + } + + private: + bool error_; + UText* utext_; + UChar32 next_; +}; + +class ReverseUnicodeCharIterator { + public: + explicit ReverseUnicodeCharIterator(const Slice& s); + + ~ReverseUnicodeCharIterator(); + + bool error() const { return error_; } + + UChar32 Get() const { return next_; } + + bool Done() const { return next_ < 0; } + + void Advance() { + next_ = utext_previous32(utext_); + } + + // Returns the position of the first byte of the current character. + int64_t Position() const { + return utext_getNativeIndex(utext_); + } + + private: + bool error_; + UText* utext_; + UChar32 next_; +}; + +// Returns true if 'c' is alphabetic. Equivalent to the \pL regex character class. +bool IsAlphaUnicode(UChar32 c); + +// Returns true if 'c' is alphanumeric. Equivalent to the \pL and \pN regex character classes. +bool IsAlphaNumUnicode(UChar32 c); + +// Returns true if 'c' is a whitespace or control character. Equivalent to the \pZ and \pC regex character classes. +bool IsSpaceControlUnicode(UChar32 c); + +#endif // VIEWFINDER_STRING_UTILS_H diff --git a/clients/shared/StringUtils.ios.mm b/clients/shared/StringUtils.ios.mm new file mode 100644 index 0000000..3903502 --- /dev/null +++ b/clients/shared/StringUtils.ios.mm @@ -0,0 +1,45 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import "StringUtils.h" + +int LocalizedCaseInsensitiveCompare(const Slice& a, const Slice& b) { + // Note that this is called frequently, so we go through the hassle of + // avoiding copying the string data. + NSString* a_str = + [[NSString alloc] + initWithBytesNoCopy:(void*)a.data() + length:a.size() + encoding:NSUTF8StringEncoding + freeWhenDone:NO]; + NSString* b_str = + [[NSString alloc] + initWithBytesNoCopy:(void*)b.data() + length:b.size() + encoding:NSUTF8StringEncoding + freeWhenDone:NO]; + return [a_str localizedCaseInsensitiveCompare:b_str]; +} + +string LocalizedNumberFormat(int value) { + NSString* s = + [NSNumberFormatter + localizedStringFromNumber:[NSNumber numberWithInt:value] + numberStyle:NSNumberFormatterDecimalStyle]; + return ToString(s); +} + +string NewUUID() { + CFUUIDRef uuid = CFUUIDCreate(NULL); + CFStringRef uuid_str = CFUUIDCreateString(NULL, uuid); + const string s = ToString((__bridge NSString*)uuid_str); + CFRelease(uuid_str); + CFRelease(uuid); + return s; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/SubscriptionManager.h b/clients/shared/SubscriptionManager.h new file mode 100644 index 0000000..3752321 --- /dev/null +++ b/clients/shared/SubscriptionManager.h @@ -0,0 +1,30 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Ben Darnell + +#ifndef VIEWFINDER_SUBSCRIPTION_MANAGER_H +#define VIEWFINDER_SUBSCRIPTION_MANAGER_H + +#import "Server.pb.h" +#import "SubscriptionMetadata.pb.h" +#import "Utils.h" + +class SubscriptionManager { + public: + struct RecordSubscription { + OpHeaders headers; + string receipt_data; + }; + + public: + virtual ~SubscriptionManager() { } + + // Returns the first queued receipt, or NULL. + virtual const RecordSubscription* GetQueuedRecordSubscription() = 0; + + // Marks the queued receipt as completed and schedules its callback. + // Records the metadata returned by the server. + virtual void CommitQueuedRecordSubscription( + const ServerSubscriptionMetadata& sub, bool success, const DBHandle& updates) = 0; +}; + +#endif // VIEWFINDER_SUBSCRIPTION_MANAGER_H diff --git a/clients/shared/SubscriptionMetadata.proto b/clients/shared/SubscriptionMetadata.proto new file mode 100644 index 0000000..444b993 --- /dev/null +++ b/clients/shared/SubscriptionMetadata.proto @@ -0,0 +1,34 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "SubscriptionMetadataPB"; + +// Local subscriptions are mostly-opaque blobs returned by iTunes/StoreKit. +// They are stored in the DB to recover from crashes between the completion +// of the itunes transaction and the call to record_subscription. +// +// TODO(ben): delete these once they have been recorded and we have seen a +// corresponding ServerSubscriptionMetadata. +// +// TODO(peter): LocalSubscriptionMetadata should be moved into +// clients/ios/Source. It is iOS specific. +message LocalSubscriptionMetadata { + optional string product = 1; + optional double timestamp = 2; + optional string receipt = 3; + optional bool recorded = 4; +} + +// The server returns the decoded subscription information via query_users. +message ServerSubscriptionMetadata { + // Note that the server transaction id will have an "itunes:" prefix not present + // in the local transaction id (or eventually some other prefix) + optional string transaction_id = 1; + optional string subscription_id = 2; + optional double timestamp = 3; + optional double expiration_ts = 4; + optional string product_type = 5; + optional double quantity = 6; + optional string payment_type = 7; +} diff --git a/clients/shared/SystemMessage.proto b/clients/shared/SystemMessage.proto new file mode 100644 index 0000000..e46b6dc --- /dev/null +++ b/clients/shared/SystemMessage.proto @@ -0,0 +1,20 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "SystemMessagePB"; + +message SystemMessage { + enum Severity { + SILENT = 0; + INFO = 1; + ATTENTION = 2; + DISABLE_NETWORK = 3; + UNKNOWN = 4; + } + optional string title = 1; + optional string body = 2; + optional string link = 3; + optional string identifier = 4; + optional Severity severity = 5 [default=UNKNOWN]; +} diff --git a/clients/shared/Timer.h b/clients/shared/Timer.h new file mode 100644 index 0000000..255c41a --- /dev/null +++ b/clients/shared/Timer.h @@ -0,0 +1,111 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_TIMER_H +#define VIEWFINDER_TIMER_H + +#include +#include "Logging.h" +#include "WallTime.h" + +using std::min; + +class WallTimer { + public: + WallTimer() + : total_time_(0), + start_time_(WallTime_Now()) { + } + + void Reset() { + total_time_ = 0; + start_time_ = 0; + } + + void Start() { + start_time_ = WallTime_Now(); + } + + void Restart() { + total_time_ = 0; + Start(); + } + + void Stop() { + if (start_time_ > 0) { + total_time_ += WallTime_Now() - start_time_; + start_time_ = 0; + } + } + + WallTime Get() const { + WallTime r = total_time_; + if (start_time_ > 0) { + r += WallTime_Now() - start_time_; + } + return r; + } + + double Milliseconds() const { + return 1000 * Get(); + } + + private: + WallTime total_time_; + WallTime start_time_; +}; + +class ScopedTimer { + public: + ScopedTimer(const string& n) + : name_(n) { + timer_.Start(); + } + ~ScopedTimer() { + LOG("%s: %0.3f sec", name_.c_str(), timer_.Get()); + } + + private: + const string name_; + WallTimer timer_; +}; + +class AverageTimer { + public: + AverageTimer(int n) + : size_(n), + count_(0), + average_(0.0) { + } + + void SetSize(int n) { + size_ = n; + int new_count = min(count_, size_); + if (new_count > 0) { + average_ = (average_ * count_) / new_count; + } + count_ = new_count; + } + + void Add(WallTime value) { + if (count_ < size_) { + ++count_; + } + average_ = (average_ * (count_ - 1) + value) / count_; + } + + WallTime Get() const { + return average_; + } + + double Milliseconds() const { + return 1000 * Get(); + } + + private: + int size_; + int count_; + WallTime average_; +}; + +#endif // VIEWFINDER_TIMER_H diff --git a/clients/shared/UserMetadata.proto b/clients/shared/UserMetadata.proto new file mode 100644 index 0000000..faa21a8 --- /dev/null +++ b/clients/shared/UserMetadata.proto @@ -0,0 +1,19 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault. + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "UsageMetadataPB"; + +message UsageCategoryMetadata { + optional int32 num_photos = 1; + optional int64 tn_size = 2; + optional int64 med_size = 3; + optional int64 full_size = 4; + optional int64 orig_size = 5; +} + +message UsageMetadata { + optional UsageCategoryMetadata owned_by = 1; + optional UsageCategoryMetadata shared_by = 2; + optional UsageCategoryMetadata visible_to = 3; +} diff --git a/clients/shared/Utils.android.cc b/clients/shared/Utils.android.cc new file mode 100644 index 0000000..cbd4430 --- /dev/null +++ b/clients/shared/Utils.android.cc @@ -0,0 +1,348 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Marc Berhault. + +#import +#import +#import +#import +#import +#import +#import +#import "Logging.h" +#import "Mutex.h" +#import "Utils.android.h" +#import "Utils.h" + +namespace { + +// We have 3 queues, the "main" queue, the "network" queue and the "priority" +// queue. The "main" and "network" queues only contain a single thread. +enum { + QUEUE_MAIN, + QUEUE_NETWORK, + QUEUE_CONCURRENT, + QUEUE_TYPES, +}; + +// Within each queue, blocks are scheduled in strict priority order. That is, +// all blocks at PRIORITY_HIGH are run before all blocks at PRIORITY_LOW. +enum { + PRIORITY_HIGH, + PRIORITY_LOW, + PRIORITY_BACKGROUND, +}; + +class ThreadDispatch { + // We maintain two maps for each queue. The ReadyMap is keyed by . The PendingMap is keyed by . When a block is ready to run, it is moved from PendingMap to + // ReadyMap where timestamp is ignored. This allows a high priority block + // scheduled shortly after a low priority block to be run before the low + // priority block. + typedef std::pair ReadyKey; + typedef std::pair PendingKey; + typedef std::map ReadyMap; + typedef std::map PendingMap; + + struct Thread { + Thread(int i) + : id(i), + thread(NULL) { + } + ~Thread() { + delete thread; + } + const int id; + std::thread* thread; + }; + + struct Queue { + Queue(const string& n, int c) + : name(n), + concurrency(c), + sequence(0), + thread_id(0) { + } + const string name; + const int concurrency; + Mutex mu; + ReadyMap ready; + PendingMap pending; + int64_t sequence; + int thread_id; + std::set threads; + }; + + public: + ThreadDispatch(JavaVM* jvm) + : jvm_(jvm), + queues_(QUEUE_TYPES, NULL) { + queues_[QUEUE_MAIN] = new Queue("main", 1); + queues_[QUEUE_NETWORK] = new Queue("network", 1); + queues_[QUEUE_CONCURRENT] = new Queue("concurrent", 3); + } + + bool IsMainThread() const { + // TODO(peter): We could replace this junk with a thread local which stores + // the queue type. + Queue* const q = queues_[QUEUE_MAIN]; + if (q->threads.empty()) { + return false; + } + Thread* const t = *q->threads.begin(); + // LOG("IsMainThread: %d", int(t->thread->get_id() == std::this_thread::get_id())); + return t->thread->get_id() == std::this_thread::get_id(); + } + + void Run(int type, int priority, double delay, const DispatchBlock& block) { + Queue* const q = queues_[type]; + MutexLock l(&q->mu); + // LOG("%s: queueing: %d %.3f", q->name, priority, delay); + const ReadyKey ready_key(priority, q->sequence++); + if (delay <= 0) { + q->ready[ready_key] = block; + } else { + const PendingKey pending_key(WallTime_Now() + delay, ready_key); + q->pending[pending_key] = block; + } + + if (q->threads.size() < q->concurrency) { + Thread* t = new Thread(q->thread_id++); + // LOG("%s/%d: starting thread", q->name, t->id); + q->threads.insert(t); + t->thread = new std::thread([this, q, t]() { + ThreadLoop(q, t); + }); + } + } + + void ThreadLoop(Queue* q, Thread* t) { + JNIEnv* env; + // TODO(peter): check return status. + jvm_->AttachCurrentThread(&env, NULL); + + q->mu.Lock(); + + for (;;) { + // LOG("%s/%d: looping: %d/%d", + // q->name, t->id, q->ready.size(), q->pending.size()); + + // Loop over the pending blocks and move them to the ready map. + const WallTime now = WallTime_Now(); + while (!q->pending.empty()) { + auto it = q->pending.begin(); + const PendingKey& key = it->first; + if (key.first > now) { + break; + } + // The block is ready to run, move it to the ready queue. + // LOG("%s/%d: ready: %d", q->name, t->id, key.second.second); + q->ready[key.second].swap(it->second); + q->pending.erase(it); + } + + if (!q->ready.empty()) { + // A block is ready to run. Run it. + auto it = q->ready.begin(); + const int64_t seq = it->first.second; + DispatchBlock block; + block.swap(it->second); + // LOG("%s/%d: running block: %d", q->name, t->id, seq); + q->ready.erase(it); + q->mu.Unlock(); + block(); + q->mu.Lock(); + // We ran a block, which might have taken a non-trivial amount of + // time. Restart the loop in order to recheck whether any pending + // blocks have become ready blocks. + continue; + } + + // No blocks were ready to run and we've already checked that the first + // pending block (if one exists) is not ready. Wait for something to do. + if (!q->pending.empty()) { + auto it = q->pending.begin(); + const PendingKey& key = it->first; + const double wait_time = key.first - now; + const int64_t seq = key.second.first; + // The next block to run will be ready to run in "wait_time" + // seconds. Wait for that amount of time, or until the sequence number + // of the first block on the queue has changed. + // LOG("%s/%d: waiting: %d %.3f", q->name, t->id, seq, wait_time); + q->mu.TimedWait(wait_time, [q, seq]() { + if (!q->ready.empty()) { + return true; + } + if (q->pending.empty()) { + return false; + } + const PendingKey& key = q->pending.begin()->first; + return key.second.first != seq; + }); + } else { + // Both queues are empty, wait until we have a block to run. + q->mu.Wait([q]() { + return !q->ready.empty() || !q->pending.empty(); + }); + } + } + + q->mu.Unlock(); + + // Threads need to be detached before terminating. + jvm_->DetachCurrentThread(); + } + + private: + JavaVM* const jvm_; + vector queues_; +}; + +ThreadDispatch* dispatch; + +struct StackCrawlState { + BacktraceData* data; + int ignore; +}; + +_Unwind_Reason_Code BacktraceFunction(_Unwind_Context* context, void* arg) { + StackCrawlState* state = reinterpret_cast(arg); + BacktraceData* data = state->data; + const uintptr_t ip = _Unwind_GetIP(context); + if (ip) { + if (state->ignore) { + state->ignore--; + } else { + data->callstack[data->frames++] = (void*)ip; + if (data->frames >= ARRAYSIZE(data->callstack)) { + return _URC_END_OF_STACK; + } + } + } + return _URC_NO_REASON; +} + +} // namespace + +string app_version; +std::function free_disk_space; +std::function total_disk_space; + +void InitDispatch(JavaVM* jvm) { + CHECK(!dispatch); + dispatch = new ThreadDispatch(jvm); +} + +string AppVersion() { + return app_version; +} + +int64_t FreeDiskSpace() { + if (!free_disk_space) { + return 0; + } + return free_disk_space(); +} + +int64_t TotalDiskSpace() { + if (!total_disk_space) { + return 0; + } + return total_disk_space(); +} + +bool dispatch_is_main_thread() { + return dispatch->IsMainThread(); +} + +void dispatch_main(const DispatchBlock& block) { + return dispatch->Run(QUEUE_MAIN, 0, 0, block); +} + +void dispatch_main_async(const DispatchBlock& block) { + return dispatch->Run(QUEUE_MAIN, 0, 0, block); +} + +void dispatch_network(const DispatchBlock& block) { + return dispatch->Run(QUEUE_NETWORK, 0, 0, block); +} + +void dispatch_high_priority(const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_HIGH, 0, block); +} + +void dispatch_low_priority(const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_LOW, 0, block); +} + +void dispatch_background(const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_BACKGROUND, 0, block); +} + +void dispatch_after_main(double delay, const DispatchBlock& block) { + return dispatch->Run(QUEUE_MAIN, 0, delay, block); +} + +void dispatch_after_network(double delay, const DispatchBlock& block) { + return dispatch->Run(QUEUE_NETWORK, 0, delay, block); +} + +void dispatch_after_high_priority(double delay, const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_HIGH, delay, block); +} + +void dispatch_after_low_priority(double delay, const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_LOW, delay, block); +} + +void dispatch_after_background(double delay, const DispatchBlock& block) { + return dispatch->Run(QUEUE_CONCURRENT, PRIORITY_BACKGROUND, delay, block); +} + +ostream& operator<<(ostream& os, const BacktraceData& bt) { + Dl_info info; + for (int i = 0; i < bt.frames; ++i) { + // The format of each line is intended to match the Android dump format: + // + // # pc () + if (!dladdr(bt.callstack[i], &info)) { + continue; + } + os << "#" << std::setw(2) << std::setfill('0') << i << " pc "; + os << std::hex << std::setw(8) << std::setfill('0') << std::noshowbase + << reinterpret_cast( + reinterpret_cast(bt.callstack[i]) - + reinterpret_cast(info.dli_fbase)); + if (info.dli_fname) { + os << " " << info.dli_fname; + } + if (info.dli_sname) { + os << " ("; + int status = 0; + char* demangled = abi::__cxa_demangle(info.dli_sname, NULL, NULL, &status); + if (demangled) { + os << demangled; + free(demangled); + } else { + os << info.dli_sname; + } + os << ")"; + } + os << "\n"; + } + return os; +} + +BacktraceData Backtrace() { + BacktraceData d; + d.frames = 0; +#if !defined(__arm__) + // TODO(peter): This doesn't currently work on arm builds because we can't + // find the _Unwind_GetIP symbol! + StackCrawlState state; + state.ignore = 1; + state.data = &d; + _Unwind_Backtrace(BacktraceFunction, &state); +#endif // !defined(__arm__) + return d; +} diff --git a/clients/shared/Utils.android.h b/clients/shared/Utils.android.h new file mode 100644 index 0000000..1b602c6 --- /dev/null +++ b/clients/shared/Utils.android.h @@ -0,0 +1,16 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_UTILS_ANDROID_H +#define VIEWFINDER_UTILS_ANDROID_H + +#include +#include + +extern string app_version; +extern std::function free_disk_space; +extern std::function total_disk_space; + +void InitDispatch(JavaVM* jvm); + +#endif // VIEWFINDER_UTILS_ANDROID_H diff --git a/clients/shared/Utils.cc b/clients/shared/Utils.cc new file mode 100644 index 0000000..b4c5aef --- /dev/null +++ b/clients/shared/Utils.cc @@ -0,0 +1,171 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "ActivityMetadata.pb.h" +#import "CommentMetadata.pb.h" +#import "EpisodeStats.pb.h" +#import "Location.pb.h" +#import "PhotoMetadata.pb.h" +#import "Placemark.pb.h" +#import "StringUtils.h" +#import "Utils.h" +#import "ViewpointMetadata.pb.h" + +ostream& operator<<(ostream& os, const ActivityId& i) { + os << i.local_id() << "[" << ServerIdFormat(i.server_id()) << "]"; + return os; +} + +ostream& operator<<(ostream& os, const CommentId& i) { + os << i.local_id() << "[" << ServerIdFormat(i.server_id()) << "]"; + return os; +} + +ostream& operator<<(ostream& os, const EpisodeId& i) { + os << i.local_id() << "[" << ServerIdFormat(i.server_id()) << "]"; + return os; +} + +ostream& operator<<(ostream& os, const EpisodeStats& s) { + os << "{\n"; + os << " posted_photos: " << s.posted_photos() << ",\n"; + os << " removed_photos: " << s.removed_photos() << "\n"; + os << "}"; + return os; +} + +ostream& operator<<(ostream& os, const PhotoId& i) { + os << i.local_id() << "[" << ServerIdFormat(i.server_id()) << "]"; + return os; +} + +ostream& operator<<(ostream& os, const ViewpointId& i) { + os << i.local_id() << "[" << ServerIdFormat(i.server_id()) << "]"; + return os; +} + +ostream& operator<<(ostream& os, const ServerIdFormat& f) { + if (f.id.empty()) { + os << "-"; + return os; + } + const string decoded = Base64HexDecode(f.id.substr(1)); + uint64_t device_id = 0; + uint64_t device_local_id = 0; + + Slice s(decoded); + // Viewpoint and operation server ids do not contain timestamps. + + // TODO(pmattis): This logic deserves to be in ServerId.{h,mm}. + if (f.id[0] != 'v' && f.id[0] != 'o') { + if (decoded.size() < 4) { + os << f.id; + return os; + } + // Skip the timestamp. + s.remove_prefix(4); + } + + device_id = Varint64Decode(&s); + device_local_id = Varint64Decode(&s); + os << f.id + << "/" << device_id + << "/" << device_local_id; + return os; +} + +ostream& operator<<(ostream& os, const Location& l) { + os << "(" << l.latitude() << ", " << l.longitude() << ")"; + return os; +} + +ostream& operator<<(ostream& os, const Placemark& p) { + vector parts; + if (p.has_sublocality()) parts.push_back(p.sublocality().c_str()); + if (p.has_locality()) parts.push_back(p.locality().c_str()); + if (p.has_state()) parts.push_back(p.state().c_str()); + if (p.has_country()) parts.push_back(p.country().c_str()); + os << Join(parts.begin(), parts.end(), ", "); + return os; +} + +ostream& operator<<(ostream& os, const google::protobuf::Message& msg) { + os << msg.DebugString(); + return os; +} + +/* + * The authors of this software are Rob Pike and Ken Thompson. + * Copyright (c) 2002 by Lucent Technologies. + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + */ + +namespace { + +typedef signed int Rune; + +enum +{ + UTFmax = 4, /* maximum bytes per rune */ + Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */ + Runeself = 0x80, /* rune and UTF sequences are the same (<) */ + Runeerror = 0xFFFD, /* decoding error in UTF */ + Runemax = 0x10FFFF, /* maximum rune value */ +}; + +} // namespace + +namespace re2 { + +int chartorune(Rune* r, const char* s); +int fullrune(const char* s, int n); + +} // namespace re2 + +int utfnlen(const char* s, int m) { + int n; + Rune rune; + const char *es = s + m; + for (n = 0; s < es; n++) { + int c = *(unsigned char*)s; + if (c < Runeself){ + if (c == '\0') + break; + s++; + continue; + } + if (!re2::fullrune(s, es - s)) + break; + s += re2::chartorune(&rune, s); + } + return n; +} + +int utfnext(Slice* s) { + if (s->empty()) { + return -1; + } + int c = *(unsigned char*)s->data(); + if (c < Runeself) { + s->remove_prefix(1); + return c; + } + if (!re2::fullrune(s->data(), s->size())) { + return -1; + } + Rune rune; + s->remove_prefix(re2::chartorune(&rune, s->data())); + return rune; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/Utils.h b/clients/shared/Utils.h new file mode 100644 index 0000000..a8ffab9 --- /dev/null +++ b/clients/shared/Utils.h @@ -0,0 +1,172 @@ +// Copyright 2011 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_UTILS_H +#define VIEWFINDER_UTILS_H + + +#import +#import +#import +#import +#import + +#ifdef OS_IOS +#import +#ifdef __OBJC__ +#import +#endif // __OBJC__ +#endif // OS_IOS + +using std::ostream; +using std::pair; +using std::string; +using std::vector; + +// Import re2::StringPiece using the more compact name Slice. +typedef re2::StringPiece Slice; + +namespace google { +namespace protobuf { +class Message; +} // namespace protobuf +} // namespace google + +class ActivityId; +class CommentId; +class EpisodeId; +class EpisodeStats; +class Location; +class PhotoId; +class Placemark; +class ViewpointId; + +struct ServerIdFormat { + ServerIdFormat(const Slice& i) + : id(i) { + } + const Slice id; +}; + +#define ARRAYSIZE(a) \ + ((sizeof(a) / sizeof(*(a))) / \ + static_cast(!(sizeof(a) % sizeof(*(a))))) + +// COMPILE_ASSERT causes a compile error about msg if expr is not true. +template struct CompileAssert {}; +#define COMPILE_ASSERT(expr, msg) \ + typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] + +typedef std::function DispatchBlock; + +// Returns true if the current thread is the main thread. +bool dispatch_is_main_thread(); + +// Runs the specified block on the main thread. If the current thread is the +// main thread, the block is run synchronously. +void dispatch_main(const DispatchBlock& block); +// Runs the specified block on the main thread. The block is always run +// asynchronously. +void dispatch_main_async(const DispatchBlock& block); + +// Runs the specified block on the network queue thread. +void dispatch_network(const DispatchBlock& block); + +// Runs the specified block on a background thread at high priority. +void dispatch_high_priority(const DispatchBlock& block); + +// Runs the specified block on a background thread at low priority. +void dispatch_low_priority(const DispatchBlock& block); + +// Runs the specified block on a background thread at background (lower than "low") priority. +void dispatch_background(const DispatchBlock& block); + +// Runs the specified block on the main/network/high/low-priority/background queue +// after the the specified delay (in seconds) has elapsed. +void dispatch_after_main(double delay, const DispatchBlock& block); +void dispatch_after_network(double delay, const DispatchBlock& block); +void dispatch_after_high_priority(double delay, const DispatchBlock& block); +void dispatch_after_low_priority(double delay, const DispatchBlock& block); +void dispatch_after_background(double delay, const DispatchBlock& block); + +#if defined(OS_IOS) + +// Runs the specified block inside of an "@autoreleasepool { }" block. +void dispatch_autoreleasepool(const DispatchBlock& block); + +#elif defined(OS_ANDROID) + +// Auto release pools are ObjectiveC-ism and do not exist on Android. +inline void dispatch_autoreleasepool(const DispatchBlock& block) { + block(); +} + +#endif // defined(OS_ANDROID) + +ostream& operator<<(ostream& os, const ActivityId& i); +ostream& operator<<(ostream& os, const CommentId& i); +ostream& operator<<(ostream& os, const EpisodeId& i); +ostream& operator<<(ostream& os, const EpisodeStats& s); +ostream& operator<<(ostream& os, const PhotoId& i); +ostream& operator<<(ostream& os, const ViewpointId& i); +ostream& operator<<(ostream& os, const ServerIdFormat& f); +ostream& operator<<(ostream& os, const Location& l); +ostream& operator<<(ostream& os, const Placemark& p); +ostream& operator<<(ostream& os, const google::protobuf::Message& msg); + +string AppVersion(); +int64_t FreeDiskSpace(); +int64_t TotalDiskSpace(); + +#ifdef OS_IOS + +extern const string kIOSVersion; +extern const string kSDKVersion; + +string TaskInfo(); +string HostInfo(); +string BuildInfo(); +string BuildRevision(); +bool IsJailbroken(); + +ostream& operator<<(ostream& os, const CGPoint& p); +ostream& operator<<(ostream& os, const CGSize& s); +ostream& operator<<(ostream& os, const CGRect& r); + +#ifdef __OBJC__ + +@class NSData; +@class NSString; + +// An output operator for NSObject and derived classes. +ostream& operator<<(ostream& os, id obj); +ostream& operator<<(ostream& os, NSString* data); +ostream& operator<<(ostream& os, NSData* data); +ostream& operator<<(ostream& os, const NSRange& r); + +#endif // __OBJC__ + +#endif // OS_IOS + +struct BacktraceData { + void* callstack[64]; + int frames; +}; + +ostream& operator<<(ostream& os, const BacktraceData& bt); + +BacktraceData Backtrace(); + +int utfnlen(const char* s, int n); +inline int utfnlen(const Slice& s) { + return utfnlen(s.data(), s.size()); +} + +// Extract and return a single utf rune from a string. +int utfnext(Slice* s); +inline int utfnext(const Slice& s) { + Slice t(s); + return utfnext(&t); +} + +#endif // VIEWFINDER_UTILS_H diff --git a/clients/shared/Utils.ios.mm b/clients/shared/Utils.ios.mm new file mode 100644 index 0000000..a9ab4c3 --- /dev/null +++ b/clients/shared/Utils.ios.mm @@ -0,0 +1,363 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import +#import +#import +#import +#import +#import "Callback.h" +#import "Format.h" +#import "LazyStaticPtr.h" +#import "Logging.h" +#import "StringUtils.h" +#import "Utils.h" + +namespace { + +typedef Callback DispatchCallback; + +class DispatchNetworkQueue { + public: + DispatchNetworkQueue() + : queue_(dispatch_queue_create("co.viewfinder.network", NULL)) { + } + + dispatch_queue_t queue() const { return queue_; } + + private: + dispatch_queue_t queue_; +}; + +LazyStaticPtr network_queue; + +void DispatchRun(void* context) { + DispatchCallback* block = reinterpret_cast(context); + (*block)(); + delete block; +} + +inline void DispatchAsync( + dispatch_queue_t queue, const DispatchBlock& block) { + // Note(peter): dispatch_async() is just a simple wrapper around + // dispatch_async_f() that copies the block and passes a function to run + // it. We do something similar, but operating on std::function<> objects + // instead. + dispatch_async_f(queue, new DispatchCallback(block), &DispatchRun); +} + +inline void DispatchAfter( + double delay, dispatch_queue_t queue, const DispatchBlock& block) { + dispatch_after_f(dispatch_time(DISPATCH_TIME_NOW, + static_cast(delay * NSEC_PER_SEC)), + queue, new DispatchCallback(block), &DispatchRun); +} + +struct DeviceInfo { + DeviceInfo() + : jailbroken(IsJailbroken()) { + } + + static bool IsJailbroken() { +#if TARGET_IPHONE_SIMULATOR + return false; +#else // !TARGET_IPHONE_SIMULATOR + // On jailbroken devices, "/bin/bash" is readable. On non-jailbroken + // devices it isn't accessible. + struct stat s; + if (stat("/bin/bash", &s) < 0) { + return false; + } + return s.st_mode & S_IFREG; +#endif // !TARGET_IPHONE_SIMULATOR + } + + bool jailbroken; +}; + +LazyStaticPtr kDeviceInfo; + +NSDictionary* DiskSpaceStats() { + NSError* error = NULL; + NSArray* paths = NSSearchPathForDirectoriesInDomains( + NSDocumentDirectory, NSUserDomainMask, YES); + NSDictionary* dictionary = + [[NSFileManager defaultManager] attributesOfFileSystemForPath: + [paths lastObject] error:&error]; + if (!dictionary) { + LOG("Unable to obtain disk space stats: %s", error); + return NULL; + } + return dictionary; +} + +// NOTE(peter): This function destructively edits the input line. +void DemangleCxxSymbols(ostream& os, char* line) { + for (;;) { + char* p = strchr(line, ' '); + if (p) { + *p = '\0'; + } + if (!p || p > line) { + int status; + char* buf = abi::__cxa_demangle(line, NULL, 0, &status); + if (buf) { + os << buf; + } else { + os << line; + } + } + if (!p) { + break; + } + os << " "; + line = p + 1; + } +} + +string GetSDKVersion() { +#ifdef __IPHONE_7_0 + return "7.0"; +#else + // This is the min-sdk version we support. + return "6.1"; +#endif +} + +} // namespace + +const string kIOSVersion = ToString([UIDevice currentDevice].systemVersion); +const string kSDKVersion = GetSDKVersion(); + +string TaskInfo() { + task_basic_info info; + mach_msg_type_number_t size = sizeof(info); + kern_return_t kerr = task_info( + mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &size); + if (kerr != KERN_SUCCESS) { + return Format("task_info() failed: %s", mach_error_string(kerr)); + } + return Format("vmem=%.1f rmem=%.1f ucpu=%d.%03d scpu=%d.%03d", + info.virtual_size / (1024.0 * 1024.0), + info.resident_size / (1024.0 * 1024.0), + info.user_time.seconds, info.user_time.microseconds / 1000, + info.system_time.seconds, info.system_time.microseconds / 1000); +} + +string HostInfo() { + host_basic_info info; + mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT; + kern_return_t kerr = host_info( + mach_host_self(), HOST_BASIC_INFO, (host_info_t)&info, &count); + if (kerr != KERN_SUCCESS) { + return Format("host_info() failed: %s", mach_error_string(kerr)); + } + return Format("memory=%.1f (%.1f) cpus=%d", + info.memory_size / (1024.0 * 1024.0), + info.max_mem / (1024.0 * 1024.0), + info.avail_cpus); +} + +string AppVersion() { + // Any changes in the version format should be reflected in the backend code: + // //viewfinder/backend/base/client_version.py + NSDictionary* d = [[NSBundle mainBundle] infoDictionary]; +#ifdef DEVELOPMENT + const char* type = ".dev"; +#elif defined(ADHOC) + const char* type = ".adhoc"; +#else // !DEVELOPMENT && !ADHOC + const char* type = ""; +#endif // !DEVELOPMENT && !ADHOC + const char* jailbroken = kDeviceInfo->jailbroken ? ".jailbroken" : ""; + return Format("%s.%s%s%s", + d[@"CFBundleShortVersionString"], + d[@"CFBundleVersion"], + type, jailbroken); +} + +string BuildInfo() { + NSString* path = + [[NSBundle mainBundle] pathForResource:@"Viewfinder-Version" + ofType:@"plist"]; + NSDictionary* d = [NSDictionary dictionaryWithContentsOfFile:path]; + return Format("built at %s: %s/%s", + d[@"BuildDate"], + d[@"BuildBranch"], + d[@"BuildRevision"]); +} + +string BuildRevision() { + NSString* path = + [[NSBundle mainBundle] pathForResource:@"Viewfinder-Version" + ofType:@"plist"]; + NSDictionary* d = [NSDictionary dictionaryWithContentsOfFile:path]; + return ToString(d[@"BuildRevision"]); +} + +int64_t FreeDiskSpace() { + NSDictionary* d = DiskSpaceStats(); + if (!d) { + return -1; + } + return [d[NSFileSystemFreeSize] longLongValue]; +} + +int64_t TotalDiskSpace() { + NSDictionary* d = DiskSpaceStats(); + if (!d) { + return -1; + } + return [d[NSFileSystemSize] longLongValue]; +} + +bool IsJailbroken() { + return kDeviceInfo->jailbroken; +} + +bool dispatch_is_main_thread() { + return [NSThread isMainThread]; +} + +void dispatch_main(const DispatchBlock& block) { + if (dispatch_is_main_thread()) { + block(); + } else { + DispatchAsync(dispatch_get_main_queue(), block); + } +} + +void dispatch_main_async(const DispatchBlock& block) { + DispatchAsync(dispatch_get_main_queue(), block); +} + +void dispatch_network(const DispatchBlock& block) { + DispatchAsync(network_queue->queue(), block); +} + +void dispatch_high_priority(const DispatchBlock& block) { + DispatchAsync( + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), + block); +} + +void dispatch_low_priority(const DispatchBlock& block) { + DispatchAsync( + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0), + block); +} + +void dispatch_background(const DispatchBlock& block) { + DispatchAsync( + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), + block); +} + +void dispatch_after_main(double delay, const DispatchBlock& block) { + DispatchAfter(delay, dispatch_get_main_queue(), block); +} + +void dispatch_after_network(double delay, const DispatchBlock& block) { + DispatchAfter(delay, network_queue->queue(), block); +} + +void dispatch_after_high_priority(double delay, const DispatchBlock& block) { + DispatchAfter(delay, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), + block); +} + +void dispatch_after_low_priority(double delay, const DispatchBlock& block) { + DispatchAfter(delay, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0), + block); +} + +void dispatch_after_background(double delay, const DispatchBlock& block) { + DispatchAfter(delay, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), + block); +} + +void dispatch_autoreleasepool(const DispatchBlock& block) { + @autoreleasepool { + block(); + } +} + +ostream& operator<<(ostream& os, const CGPoint& p) { + os << "<" << p.x << " " << p.y << ">"; + return os; +} + +ostream& operator<<(ostream& os, const CGSize& s) { + os << "<" << s.width << " " << s.height << ">"; + return os; +} + +ostream& operator<<(ostream& os, const CGRect& r) { + os << "<" << r.origin.x << " " << r.origin.y + << " " << r.size.width << " " << r.size.height << ">"; + return os; +} + +// An output operator for NSObject and derived classes. +ostream& operator<<(ostream& os, id obj) { + if (!obj) { + os << "(null)"; + } else { + os << [[obj description] UTF8String]; + } + return os; +} + +ostream& operator<<(ostream& os, NSString* str) { + if (!str) { + os << "(null)"; + } else { + os << [str UTF8String]; + } + return os; +} + +ostream& operator<<(ostream& os, NSData* data) { + if (!data) { + os << "(null)"; + } else { + os.write((const char*)data.bytes, data.length); + } + return os; +} + +ostream& operator<<(ostream& os, const NSRange& r) { + os << r.location << "," << r.length; + return os; +} + +ostream& operator<<(ostream& os, const BacktraceData& bt) { + // Skip the first frame which is internal to Backtrace(). + char** strs = backtrace_symbols(bt.callstack, bt.frames); + for (int i = 1; i < bt.frames; ++i) { + DemangleCxxSymbols(os, strs[i]); + os << "\n"; + } + free(strs); + return os; +} + +BacktraceData Backtrace() { + // NOTE(peter): The obvious design would be to have a Backtrace class and put + // this initialization in the constructor. Unfortunately, the compiler + // creates 1 or 2 stackframes for that constructor depending on optimization + // levels. Using a Backtrace() function deterministically adds a single stack + // frame which can be skipped in the operator<<() method. + BacktraceData d; + d.frames = backtrace(d.callstack, 64); + return d; +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ViewpointMetadata.proto b/clients/shared/ViewpointMetadata.proto new file mode 100644 index 0000000..d2db61c --- /dev/null +++ b/clients/shared/ViewpointMetadata.proto @@ -0,0 +1,58 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +import "ContentIds.proto"; +import "QueueMetadata.proto"; + +option java_package = "co.viewfinder.proto"; +option java_outer_classname = "ViewpointMetadataPB"; + +message CoverPhoto { + optional PhotoId photo_id = 1; + optional EpisodeId episode_id = 2; +} + +message ViewpointMetadata { + optional ViewpointId id = 1; + optional int64 user_id = 2; + optional int64 sharing_user_id = 3; + optional string title = 4; + optional string description = 5; + optional string name = 6; + optional CoverPhoto cover_photo = 11; + // Refer to backend/db/viewpoint.py for type specifications. + // As of the time of this writing there are: + // ['default', 'event', 'system'] + optional string type = 7; + optional int64 update_seq = 8; + optional int64 viewed_seq = 9; + optional QueueMetadata queue = 10; + + // Bits indicating labels on the "follower" relation. + optional bool label_admin = 20; + optional bool label_autosave = 26; + optional bool label_contribute = 21; + optional bool DEPRECATED_label_personal = 22; + optional bool label_hidden = 23; + optional bool label_removed = 24; + optional bool label_muted = 25; + optional bool label_unrevivable = 27; + optional bool label_error = 30; + + // The following are client-side-only attributes. + + // Provisional activities are not uploaded to the server until they become + // permanent. + optional bool provisional = 45; + // Does this viewpoint need to be updated on the server? + optional bool update_metadata = 40; + optional bool update_follower_metadata = 41; + optional bool update_remove = 44; + optional bool update_viewed_seq = 47; + + repeated string indexed_terms = 46; + + // Used to display "new-content" indicators next to activities. + optional int64 DEPRECATED_last_viewed_seq = 42; + optional double DECPRECATED_last_viewed_timestamp = 43; +} diff --git a/clients/shared/ViewpointTable.cc b/clients/shared/ViewpointTable.cc new file mode 100644 index 0000000..7afb0e9 --- /dev/null +++ b/clients/shared/ViewpointTable.cc @@ -0,0 +1,1953 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import "ActivityTable.h" +#import "AppState.h" +#import "AsyncState.h" +#import "CommentTable.h" +#import "ContactManager.h" +#import "DayTable.h" +#import "EpisodeTable.h" +#import "FullTextIndex.h" +#import "IdentityManager.h" +#import "NetworkManager.h" +#import "NetworkQueue.h" +#import "PeopleRank.h" +#import "PhotoTable.h" +#import "PlacemarkHistogram.h" +#import "ServerId.h" +#import "StringUtils.h" +#import "ViewpointTable.h" + +const string ViewpointTable::kViewpointTokenPrefix = "_vp"; + +namespace { + +const int kViewpointFSCKVersion = 2; + +const WallTime kClawbackGracePeriod = 7 * 24 * 60 * 60; // 1 week + +const string kFollowerViewpointKeyPrefix = DBFormat::follower_viewpoint_key(""); +const string kViewpointFollowerKeyPrefix = DBFormat::viewpoint_follower_key(""); +const string kViewpointGCKeyPrefix = DBFormat::viewpoint_gc_key(""); +const string kViewpointSelectionKeyPrefix = DBFormat::viewpoint_selection_key(""); +const string kViewpointScrollOffsetKeyPrefix = DBFormat::viewpoint_scroll_offset_key(""); +const string kHasUserCreatedViewpointKey = DBFormat::metadata_key("has_user_created_viewpoint"); +const string kViewpointIndexName = "vp"; +const string kViewpointGCCommitTrigger = "ViewpontTableGC"; + +const DBRegisterKeyIntrospect kFollowerViewpointKeyIntrospect( + kFollowerViewpointKeyPrefix, [](Slice key) { + int64_t follower_id; + int64_t viewpoint_id; + if (!DecodeFollowerViewpointKey(key, &follower_id, &viewpoint_id)) { + return string(); + } + return string(Format("%d/%d", follower_id, viewpoint_id)); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointKeyIntrospect( + DBFormat::viewpoint_key(), NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kViewpointServerKeyIntrospect( + DBFormat::viewpoint_server_key(), NULL, [](Slice value) { + return value.ToString(); + }); + +const DBRegisterKeyIntrospect kViewpointFollowerKeyIntrospect( + kViewpointFollowerKeyPrefix, [](Slice key) { + int64_t viewpoint_id; + int64_t follower_id; + if (!DecodeViewpointFollowerKey(key, &viewpoint_id, &follower_id)) { + return string(); + } + return string(Format("%d/%d", viewpoint_id, follower_id)); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointGCKeyIntrospect( + kViewpointGCKeyPrefix, [](Slice key) { + int64_t viewpoint_id; + WallTime expiration; + if (!DecodeViewpointGCKey(key, &viewpoint_id, &expiration)) { + return string(); + } + return string(Format("%d/%s", viewpoint_id, DBIntrospect::timestamp(expiration))); + }, NULL); + +const DBRegisterKeyIntrospect kViewpointSelectionKeyIntrospect( + kViewpointSelectionKeyPrefix, NULL, [](Slice value) { + return DBIntrospect::FormatProto(value); + }); + +const DBRegisterKeyIntrospect kViewpointScrollOffsetKeyIntrospect( + kViewpointScrollOffsetKeyPrefix, + [](Slice key) { + int64_t viewpoint_id; + if (!DecodeViewpointScrollOffsetKey(key, &viewpoint_id)) { + return string(); + } + return string(Format("%d", viewpoint_id)); + }, + [](Slice value) { + return value.ToString(); + }); + +// Creates a local activity as a placeholder for display. The metadata +// will eventually be replaced on a notification from the server after +// the client uploads the activity via a share_new, share_existing, +// add_followers, post_comment, etc. operation. +ActivityHandle NewLocalActivity( + AppState* state, WallTime timestamp, + const ViewpointHandle& vh, const DBHandle& updates) { + ActivityHandle ah = state->activity_table()->NewActivity(updates); + ah->Lock(); + ah->set_timestamp(timestamp); + ah->set_user_id(state->user_id()); + ah->mutable_activity_id()->set_server_id( + EncodeActivityId(state->device_id(), ah->activity_id().local_id(), + ah->timestamp())); + ah->mutable_viewpoint_id()->CopyFrom(vh->id()); + + // NOTE: we don't set the activity's update_seq value as there's no + // way to know in advance the correct value that the server will + // assign when the activity is uploaded. When the uploaded activity + // is queried on a subsequent notification, the correct value will + // be set. + ah->set_upload_activity(true); + + return ah; +} + +// Creates a local activity as a placeholder for upload, but without +// an explicit viewpoint, as the activity is meant for the default +// viewpoint. +ActivityHandle NewLocalActivity( + AppState* state, WallTime timestamp, const DBHandle& updates) { + ActivityHandle ah = state->activity_table()->NewActivity(updates); + ah->Lock(); + ah->set_timestamp(timestamp); + ah->set_user_id(state->user_id()); + ah->mutable_activity_id()->set_server_id( + EncodeActivityId(state->device_id(), ah->activity_id().local_id(), + ah->timestamp())); + ah->set_upload_activity(true); + + return ah; +} + +// Fits the specified photo ids where possible into "existing_episodes"; +// otherwise, creates new derivative episodes as necessary to hold photos. +// A description of episode id / photo ids for each is stored in +// *share_episodes. +bool AddPhotosToActivity( + AppState* state, const vector& existing_episodes, + ShareEpisodes* share_episodes, const ViewpointHandle& vh, + const PhotoSelectionVec& photo_ids, const DBHandle& updates) { + vector episodes; + + for (int i = 0; i < photo_ids.size(); ++i) { + const int64_t photo_id = photo_ids[i].photo_id; + const int64_t episode_id = photo_ids[i].episode_id; + + PhotoHandle ph = state->photo_table()->LoadPhoto(photo_id, updates); + if (!ph.get()) { + // Unable to find the photo. + return false; + } + + // Find the existing episode structure if it exists. This could be + // optimized, but should be plenty fast even when sharing hundreds of + // photos. + ActivityMetadata::Episode* e = NULL; + EpisodeHandle eh; + for (int j = 0; j < episodes.size(); ++j) { + if (episodes[j]->parent_id().local_id() == episode_id) { + eh = episodes[j]; + e = share_episodes->Mutable(j); + break; + } + } + if (!e) { + // Check the existing_episodes vector. + for (int j = 0; j < existing_episodes.size(); ++j) { + if (existing_episodes[j]->parent_id().local_id() == episode_id) { + eh = existing_episodes[j]; + break; + } + } + if (!eh.get()) { + // Create a new episode, inheriting bits of state from the parent + // episode. + EpisodeHandle parent = state->episode_table()->LoadEpisode(episode_id, updates); + if (!parent.get()) { + // Unable to find episode photo is being shared from. + return false; + } + + eh = state->episode_table()->NewEpisode(updates); + eh->Lock(); + eh->set_user_id(state->user_id()); + eh->set_timestamp(parent->timestamp()); + eh->set_publish_timestamp(WallTime_Now()); + eh->set_upload_episode(true); + eh->mutable_id()->set_server_id( + EncodeEpisodeId(state->device_id(), eh->id().local_id(), + eh->timestamp())); + eh->mutable_parent_id()->CopyFrom(parent->id()); + if (vh.get()) { + eh->mutable_viewpoint_id()->CopyFrom(vh->id()); + } + } else { + eh->Lock(); + } + episodes.push_back(eh); + + e = share_episodes->Add(); + e->mutable_episode_id()->CopyFrom(eh->id()); + } + + e->add_photo_ids()->CopyFrom(ph->id()); + eh->AddPhoto(ph->id().local_id()); + } + + for (int i = 0; i < episodes.size(); ++i) { + episodes[i]->SaveAndUnlock(updates); + } + return true; +} + +bool AddPhotosToShareActivity( + AppState* state, ShareEpisodes* share_episodes, const ViewpointHandle& vh, + const PhotoSelectionVec& photo_ids, const DBHandle& updates) { + // List the existing episodes in the viewpoint. + vector existing_episodes; + vh->ListEpisodes(&existing_episodes); + + return AddPhotosToActivity(state, existing_episodes, share_episodes, vh, photo_ids, updates); +} + +bool AddPhotosToSaveActivity( + AppState* state, ShareEpisodes* share_episodes, + const PhotoSelectionVec& photo_ids, const DBHandle& updates) { + // Get list of all episodes which have parent id equal to episode + // ids specified in "photo_ids" and are part of the default + // viewpoint. We filter this list to contain only episodes which are + // local or part of the default viewpoint. This comprises the list + // of "existing" episodes to which we want to add any overlapping + // photo ids. + vector existing_episodes; + for (int i = 0; i < photo_ids.size(); ++i) { + const int64_t episode_id = photo_ids[i].episode_id; + + vector child_ids; + state->episode_table()->ListEpisodesByParentId(episode_id, &child_ids, updates); + for (int j = 0; j < child_ids.size(); ++j) { + EpisodeHandle eh = state->episode_table()->LoadEpisode(child_ids[j], updates); + if (eh.get()) { + ViewpointHandle vh = state->viewpoint_table()->LoadViewpoint(eh->viewpoint_id(), updates); + if (!vh.get() || vh->is_default()) { + DCHECK_EQ(eh->user_id(), state->user_id()); + existing_episodes.push_back(eh); + } + } + } + } + + ViewpointHandle vh; + return AddPhotosToActivity(state, existing_episodes, share_episodes, vh, photo_ids, updates); +} + +bool AddContactsToAddFollowersActivity( + const ActivityHandle& ah, const vector& contacts) { + for (int i = 0; i < contacts.size(); ++i) { + ah->mutable_add_followers()->add_contacts()->CopyFrom(contacts[i]); + } + return true; +} + +bool AddContactsToShareNewActivity( + const ActivityHandle& ah, const vector& contacts) { + for (int i = 0; i < contacts.size(); ++i) { + ah->mutable_share_new()->add_contacts()->CopyFrom(contacts[i]); + } + return true; +} + +} // namespace + +string EncodeFollowerViewpointKey(int64_t follower_id, int64_t viewpoint_id) { + string s; + OrderedCodeEncodeInt64Pair(&s, follower_id, viewpoint_id); + return DBFormat::follower_viewpoint_key(s); +} + +string EncodeViewpointFollowerKey(int64_t viewpoint_id, int64_t follower_id) { + string s; + OrderedCodeEncodeInt64Pair(&s, viewpoint_id, follower_id); + return DBFormat::viewpoint_follower_key(s); +} + +string EncodeViewpointGCKey(int64_t viewpont_id, WallTime expiration) { + string s; + OrderedCodeEncodeVarint64(&s, viewpont_id); + OrderedCodeEncodeVarint32(&s, expiration); + return DBFormat::viewpoint_gc_key(s); +} + +string EncodeViewpointScrollOffsetKey(int64_t viewpoint_id) { + string s; + OrderedCodeEncodeVarint64(&s, viewpoint_id); + return DBFormat::viewpoint_scroll_offset_key(s); +} + +bool DecodeFollowerViewpointKey(Slice key, int64_t* follower_id, int64_t* viewpoint_id) { + if (!key.starts_with(kFollowerViewpointKeyPrefix)) { + return false; + } + key.remove_prefix(kFollowerViewpointKeyPrefix.size()); + OrderedCodeDecodeInt64Pair(&key, follower_id, viewpoint_id); + return true; +} + +bool DecodeViewpointFollowerKey(Slice key, int64_t* viewpoint_id, int64_t* follower_id) { + if (!key.starts_with(kViewpointFollowerKeyPrefix)) { + return false; + } + key.remove_prefix(kViewpointFollowerKeyPrefix.size()); + OrderedCodeDecodeInt64Pair(&key, viewpoint_id, follower_id); + return true; +} + +bool DecodeViewpointGCKey(Slice key, int64_t* viewpoint_id, WallTime* expiration) { + if (!key.starts_with(kViewpointGCKeyPrefix)) { + return false; + } + key.remove_prefix(kViewpointGCKeyPrefix.size()); + *viewpoint_id = OrderedCodeDecodeVarint64(&key); + *expiration = OrderedCodeDecodeVarint32(&key); + return true; +} + +bool DecodeViewpointScrollOffsetKey(Slice key, int64_t* viewpoint_id) { + if (!key.starts_with(kViewpointScrollOffsetKeyPrefix)) { + return false; + } + key.remove_prefix(kViewpointScrollOffsetKeyPrefix.size()); + *viewpoint_id = OrderedCodeDecodeVarint64(&key); + return true; +} + + +//// +// ViewpointTable_Viewpoint + +const string ViewpointTable_Viewpoint::kTypeDefault = "default"; +const double ViewpointTable_Viewpoint::kViewpointGCExpirationSeconds = 60 * 60 * 24; // 1 day + +ViewpointTable_Viewpoint::ViewpointTable_Viewpoint( + AppState* state, const DBHandle& db, int64_t id) + : state_(state), + db_(db), + disk_label_removed_(true) { + mutable_id()->set_local_id(id); +} + +void ViewpointTable_Viewpoint::MergeFrom(const ViewpointMetadata& m) { + // Some assertions that immutable properties don't change. + if (has_user_id() && m.has_user_id()) { + DCHECK_EQ(user_id(), m.user_id()); + } + if (m.has_cover_photo()) { + // Clear existing cover photo in case client has just a + // local id and server is sending just a server id--we don't + // want any franken-merging in the event the cover photo changes. + clear_cover_photo(); + } + + // Ratchet the update/viewed sequence numbers to allow only monotonic + // increases. In the case of viewed_seq, the client may have updated + // locally only to have a concurrent query via viewpoint invalidation + // overwrite the local value. + const int64_t max_update_seq = std::max(update_seq(), m.update_seq()); + const int64_t max_viewed_seq = std::max(viewed_seq(), m.viewed_seq()); + + ViewpointMetadata::MergeFrom(m); + + set_update_seq(max_update_seq); + set_viewed_seq(max_viewed_seq); +} + +void ViewpointTable_Viewpoint::MergeFrom(const ::google::protobuf::Message&) { + DIE("MergeFrom(Message&) should not be used"); +} + + +void ViewpointTable_Viewpoint::AddFollower(int64_t follower_id) { + EnsureFollowerState(); + if (!ContainsKey(*followers_, follower_id) || + (*followers_)[follower_id] == REMOVED) { + (*followers_)[follower_id] = ADDED; + } +} + +void ViewpointTable_Viewpoint::RemoveFollower(int64_t follower_id) { + EnsureFollowerState(); + (*followers_)[follower_id] = REMOVED; +} + +int ViewpointTable_Viewpoint::CountFollowers() { + EnsureFollowerState(); + return followers_->size(); +} + +void ViewpointTable_Viewpoint::ListFollowers(vector* follower_ids) { + EnsureFollowerState(); + for (FollowerStateMap::iterator iter(followers_->begin()); + iter != followers_->end(); + ++iter) { + follower_ids->push_back(iter->first); + } +} + +void ViewpointTable_Viewpoint::GetRemovableFollowers( + std::unordered_set* removable) { + std::unordered_map removable_map; + + vector follower_ids; + ListFollowers(&follower_ids); + for (int i = 0; i < follower_ids.size(); ++i) { + removable_map[follower_ids[i]] = false; + } + removable_map[state_->user_id()] = true; + + // Next, any users who were added during share_new or add_followers + // activities originated by the current user less than 7 days ago are + // added to removable_map=true. + std::unordered_map merged; // source_id -> target_id + for (ScopedPtr iter( + state_->activity_table()->NewViewpointActivityIterator( + local_id(), std::numeric_limits::max(), true, db_)); + !iter->done(); + iter->Prev()) { + ActivityHandle ah = iter->GetActivity(); + + // Keep track of all merged accounts. + if (ah->has_merge_accounts()) { + merged[ah->merge_accounts().source_user_id()] = ah->merge_accounts().target_user_id(); + } + + // Skip activities not owned by the current user or are more than + // the removable time limit. + if (ah->user_id() != state_->user_id() || + (state_->WallTime_Now() - ah->timestamp()) > kClawbackGracePeriod) { + continue; + } + + // add_followers and share_new activities have a list of users. + const ::google::protobuf::RepeatedPtrField* contacts = NULL; + if (ah->has_add_followers()) { + contacts = &ah->add_followers().contacts(); + } else if (ah->has_share_new()) { + contacts = &ah->share_new().contacts(); + } + if (contacts) { + for (int i = 0; i < contacts->size(); ++i) { + ContactMetadata cm = contacts->Get(i); + if (cm.has_user_id()) { + int64_t user_id = cm.user_id(); + if (ContainsKey(merged, user_id)) { + user_id = merged[user_id]; + } + if (ContainsKey(removable_map, user_id)) { + removable_map[user_id] = true; + } + } + } + } + } + + removable->clear(); + for (std::unordered_map::iterator iter = removable_map.begin(); + iter != removable_map.end(); + ++iter) { + if (iter->second) { + removable->insert(iter->first); + } + } +} + +void ViewpointTable_Viewpoint::ListEpisodes( + vector* episodes) { + std::unordered_set unique_server_ids; + for (ScopedPtr iter( + state_->activity_table()->NewViewpointActivityIterator( + id().local_id(), 0, false, db_)); + !iter->done(); + iter->Next()) { + ActivityHandle ah = iter->GetActivity(); + if (!ah.get()) { + continue; + } + const ShareEpisodes* share_episodes = ah->GetShareEpisodes(); + if (!share_episodes) { + continue; + } + + for (int i = 0; i < share_episodes->size(); ++i) { + const ActivityMetadata::Episode& episode = share_episodes->Get(i); + const string server_id = episode.episode_id().server_id(); + if (!ContainsKey(unique_server_ids, server_id)) { + unique_server_ids.insert(server_id); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(server_id, db_); + if (eh.get()) { + episodes->push_back(eh); + } + } + } + } +} + +EpisodeHandle ViewpointTable_Viewpoint::GetAnchorEpisode(PhotoHandle* ph_ptr) { + // TODO(spencer): when cover photo can be updated, the anchor + // episode will be set as viewpoint metadata. + + // As a fallback, iterate through activities until the first share + // with photos. These loops look daunting, but in most cases, it's + // just going to grab the very first photo, which will be from the + // first episode of the first share. + for (ScopedPtr iter( + state_->activity_table()->NewViewpointActivityIterator( + id().local_id(), 0, false, db_)); + !iter->done(); + iter->Next()) { + ActivityHandle ah = iter->GetActivity(); + if (!ah.get() || + (!ah->has_share_new() && !ah->has_share_existing())) { + continue; + } + + const ShareEpisodes* episodes = ah->GetShareEpisodes(); + for (int i = 0; i < episodes->size(); ++i) { + EpisodeHandle eh = state_->episode_table()->LoadEpisode( + episodes->Get(i).episode_id(), db_); + if (!eh.get()) continue; + + // Exclude unshared ids. + vector unshared_ids; + eh->ListUnshared(&unshared_ids); + std::unordered_set unshared_set( + unshared_ids.begin(), unshared_ids.end()); + + // Get first shared photo which we successfully load. + for (int j = 0; j < episodes->Get(i).photo_ids_size(); ++j) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(episodes->Get(i).photo_ids(j), db_); + if (ph.get() && !ContainsKey(unshared_set, ph->id().local_id())) { + if (ph_ptr) { + *ph_ptr = ph; + } + return eh; + } + } + } + } + return EpisodeHandle(); +} + +bool ViewpointTable_Viewpoint::GetCoverPhoto( + int64_t* photo_id, int64_t* episode_id, WallTime* timestamp, + float* aspect_ratio) { + if (!has_cover_photo()) { + return false; + } + PhotoHandle ph = + state_->photo_table()->LoadPhoto(cover_photo().photo_id(), db_); + EpisodeHandle eh = + state_->episode_table()->LoadEpisode(cover_photo().episode_id(), db_); + if (!ph.get() || ph->label_error() || + !eh.get() || eh->IsUnshared(ph->id().local_id())) { + return false; + } + + *photo_id = ph->id().local_id(); + *episode_id = eh->id().local_id(); + *timestamp = ph->timestamp(); + *aspect_ratio = ph->aspect_ratio(); + return true; +} + +string ViewpointTable_Viewpoint::FormatTitle(bool shorten, bool normalize_whitespace) { + if (has_title() && !title().empty()) { + return normalize_whitespace ? NormalizeWhitespace(title()) : title(); + } + return DefaultTitle(); +} + +string ViewpointTable_Viewpoint::DefaultTitle() { + // Get anchor episode to craft a default title. + EpisodeHandle eh = GetAnchorEpisode(NULL); + if (eh.get()) { + Location location; + Placemark placemark; + if (eh->GetLocation(&location, &placemark)) { + // Use shortened format of location. + string loc_str; + state_->placemark_histogram()->FormatLocation(location, placemark, true, &loc_str); + return loc_str; + } + } + return "Untitled"; +} + +void ViewpointTable_Viewpoint::InvalidateEpisodes(const DBHandle& updates) { + if (is_default()) { + // Don't invalidate trapdoors for the default viewpoint. + return; + } + + vector episodes; + ListEpisodes(&episodes); + for (int i = 0; i < episodes.size(); ++i) { + episodes[i]->Invalidate(updates); + } +} + +float ViewpointTable_Viewpoint::GetGCExpiration() { + return state_->WallTime_Now() + (label_unrevivable() ? 0 : kViewpointGCExpirationSeconds); +} + +void ViewpointTable_Viewpoint::SaveHook(const DBHandle& updates) { + // If the viewpoint is being removed (or added back), make sure to + // update the appropriate follower groups. + if (label_removed() != disk_label_removed_) { + EnsureFollowerState(); + } + + if (followers_.get()) { + vector original; // original list of followers + vector current; // current list of followers + vector removed; // keep track of followers to remove from map + vector added; // keep track of any added followers + // Persist any added followers. + for (FollowerStateMap::iterator iter(followers_->begin()); + iter != followers_->end(); ) { + FollowerStateMap::iterator cur(iter++); + const int64_t follower_id = cur->first; + if (cur->second == ADDED) { + updates->Put(EncodeViewpointFollowerKey(id().local_id(), follower_id), string()); + updates->Put(EncodeFollowerViewpointKey(follower_id, id().local_id()), string()); + cur->second = LOADED; + added.push_back(follower_id); + current.push_back(follower_id); + } else if (cur->second == REMOVED) { + original.push_back(follower_id); + updates->Delete(EncodeViewpointFollowerKey(id().local_id(), follower_id)); + updates->Delete(EncodeFollowerViewpointKey(follower_id, id().local_id())); + removed.push_back(follower_id); + // If the removed follower is the user, ensure that + // removed/unrevivable labels are set. + if (follower_id == state_->user_id()) { + set_label_removed(true); + set_label_unrevivable(true); + } + } else { + DCHECK_EQ(cur->second, LOADED); + original.push_back(follower_id); + current.push_back(follower_id); + } + } + // Erase removed followers from map. + for (int i = 0; i < removed.size(); ++i) { + followers_->erase(removed[i]); + } + + // Update the follower groups if applicable. + if (!added.empty() || !removed.empty() || + disk_label_removed_ != label_removed()) { + if (type() != "system") { + // Don't count followers of system viewpoints. + if (!disk_label_removed_) { + state_->people_rank()->RemoveViewpoint(id().local_id(), original, updates); + } + if (!label_removed()) { + state_->people_rank()->AddViewpoint(id().local_id(), current, updates); + } + } + disk_label_removed_ = label_removed(); + } + } + + // If removed label is set, make sure to add this viewpoint to + // the garbage collection queue. + if (label_removed()) { + const WallTime expiration = GetGCExpiration(); + updates->Put(EncodeViewpointGCKey(id().local_id(), expiration), string()); + + // If unrevivable, schedule immediate processing of the queue on commit. + if (label_unrevivable()) { + AppState* state = state_; + updates->AddCommitTrigger(kViewpointGCCommitTrigger, [state] { + dispatch_after_main(0, [state] { + state->viewpoint_table()->ProcessGCQueue(); + }); + }); + } + } + + // If we have changes to the cover photo, perform necessary invalidations. + if (cover_photo().photo_id().server_id() != orig_cover_photo_.photo_id().server_id() || + cover_photo().episode_id().server_id() != orig_cover_photo_.episode_id().server_id()) { + EpisodeHandle old_eh; + if (orig_cover_photo_.has_episode_id()) { + old_eh = state_->episode_table()->LoadEpisode(orig_cover_photo_.episode_id(), updates); + if (old_eh.get()) { + old_eh->Invalidate(updates); + } + } + EpisodeHandle eh = state_->episode_table()->LoadEpisode(cover_photo().episode_id(), updates); + if (eh.get()) { + eh->Invalidate(updates); + } + orig_cover_photo_.CopyFrom(cover_photo()); + } + + typedef ContentTable::Content Content; + ViewpointHandle vh (reinterpret_cast(this)); + state_->net_queue()->QueueViewpoint(vh, updates); + + const string new_day_table_fields = GetDayTableFields(); + if (day_table_fields_ != new_day_table_fields) { + // If not the default viewpoint, invalidate so changes to metadata + // (e.g. title, labels, unviewed, etc.) are visible. + if (!is_default()) { + state_->day_table()->InvalidateViewpoint(vh, updates); + } + day_table_fields_ = new_day_table_fields; + } +} + +void ViewpointTable_Viewpoint::DeleteHook(const DBHandle& updates) { + // Delete all activities. + for (ScopedPtr iter( + state_->activity_table()->NewViewpointActivityIterator( + local_id(), 0, false, updates)); + !iter->done(); + iter->Next()) { + ActivityHandle ah = iter->GetActivity(); + if (ah.get()) { + ah->Lock(); + ah->DeleteAndUnlock(updates); + } + } + + EnsureFollowerState(); + if (followers_.get()) { + vector original; // original list of followers + // Persist any added followers. + for (FollowerStateMap::iterator iter(followers_->begin()); + iter != followers_->end(); + iter++) { + const int64_t follower_id = iter->first; + original.push_back(follower_id); + updates->Delete(EncodeViewpointFollowerKey(id().local_id(), follower_id)); + updates->Delete(EncodeFollowerViewpointKey(follower_id, id().local_id())); + } + + // Update the follower groups. Don't count followers of system + // viewpoints. + if (type() != "system") { + state_->people_rank()->RemoveViewpoint(id().local_id(), original, updates); + } + } + + // Remove scroll offset key. + state_->db()->Delete(EncodeViewpointScrollOffsetKey(id().local_id())); + + typedef ContentTable::Content Content; + ViewpointHandle vh (reinterpret_cast(this)); + state_->net_queue()->DequeueViewpoint(vh, updates); + state_->day_table()->InvalidateViewpoint(vh, updates); +} + +string ViewpointTable_Viewpoint::GetDayTableFields() const { + ViewpointMetadata m(*this); + m.clear_queue(); + m.clear_update_metadata(); + m.clear_update_follower_metadata(); + m.clear_update_remove(); + m.clear_update_viewed_seq(); + return m.SerializeAsString(); +} + +bool ViewpointTable_Viewpoint::Load() { + if (has_cover_photo()) { + orig_cover_photo_.CopyFrom(cover_photo()); + } + day_table_fields_ = GetDayTableFields(); + disk_label_removed_ = label_removed(); + return true; +} + +void ViewpointTable_Viewpoint::EnsureFollowerState() { + if (followers_.get()) { + return; + } + followers_.reset(new FollowerStateMap); + for (DB::PrefixIterator iter(db_, EncodeViewpointFollowerKey(id().local_id(), 0)); + iter.Valid(); + iter.Next()) { + int64_t viewpoint_id; + int64_t follower_id; + if (DecodeViewpointFollowerKey(iter.key(), &viewpoint_id, &follower_id)) { + (*followers_)[follower_id] = LOADED; + } + } +} + + +//// +// ViewpointTable + +ViewpointTable::ViewpointTable(AppState* state) + : ContentTable(state, + DBFormat::viewpoint_key(), + DBFormat::viewpoint_server_key(), + kViewpointFSCKVersion, + DBFormat::metadata_key("viewpoint_table_fsck")), + viewpoint_index_(new FullTextIndex(state_, kViewpointIndexName)) { + state_->app_did_become_active()->Add([this] { + ProcessGCQueue(); + }); +} + +ViewpointTable::~ViewpointTable() { +} + +ViewpointHandle ViewpointTable::LoadViewpoint(const ViewpointId& id, const DBHandle& db) { + ViewpointHandle vh; + if (id.has_local_id()) { + vh = LoadViewpoint(id.local_id(), db); + } + if (!vh.get() && id.has_server_id()) { + vh = LoadViewpoint(id.server_id(), db); + } + return vh; +} + +void ViewpointTable::CanonicalizeViewpointId( + ViewpointId* vp_id, const DBHandle& updates) { + // Do not try to canonicalize if there are no available ids. + if (!vp_id->has_server_id() && !vp_id->has_local_id()) { + LOG("cannot canonicalize empty viewpoint id"); + return; + } + ViewpointHandle vh = LoadViewpoint(*vp_id, updates); + if (!vh.get()) { + vh = NewViewpoint(updates); + vh->Lock(); + vh->mutable_id()->set_server_id(vp_id->server_id()); + { + // The "default" viewpoint has special handling. We need to set + // ViewpointMetadata::type() appropriately when synthesizing a viewpoint. + int64_t device_id; + int64_t device_local_id; + if (DecodeViewpointId(vp_id->server_id(), &device_id, &device_local_id)) { + if (device_local_id == 0) { + vh->set_type(vh->kTypeDefault); + } + } + // Create a full invalidation for the viewpoint. + InvalidateFull(vp_id->server_id(), updates); + } + vh->SaveAndUnlock(updates); + vp_id->set_local_id(vh->id().local_id()); + } else if (!vp_id->has_local_id()) { + vp_id->set_local_id(vh->id().local_id()); + } +} + +void ViewpointTable::ListViewpointsForPhotoId( + int64_t photo_id, vector* viewpoint_ids, const DBHandle& db) { + std::set unique_viewpoint_ids; + vector episode_ids; + state_->episode_table()->ListEpisodes(photo_id, &episode_ids, db); + for (int i = 0; i < episode_ids.size(); ++i) { + EpisodeHandle eh = state_->episode_table()->LoadEpisode(episode_ids[i], db); + if (!eh.get() || !eh->has_viewpoint_id()) { + continue; + } + ViewpointHandle vh = LoadViewpoint(eh->viewpoint_id(), db); + if (vh.get() && !vh->is_default() && !vh->label_removed()) { + unique_viewpoint_ids.insert(vh->id().local_id()); + } + } + viewpoint_ids->clear(); + viewpoint_ids->insert(viewpoint_ids->begin(), + unique_viewpoint_ids.begin(), unique_viewpoint_ids.end()); +} + +void ViewpointTable::ListViewpointsForUserId( + int64_t user_id, vector* viewpoint_ids, const DBHandle& db) { + for (DB::PrefixIterator iter(db, EncodeFollowerViewpointKey(user_id, 0)); + iter.Valid(); + iter.Next()) { + int64_t follower_id; + int64_t viewpoint_id; + if (DecodeFollowerViewpointKey(iter.key(), &follower_id, &viewpoint_id)) { + ViewpointHandle vh = LoadViewpoint(viewpoint_id, db); + if (vh.get() && !vh->is_default() && !vh->label_removed()) { + viewpoint_ids->push_back(viewpoint_id); + } + } + } +} + +bool ViewpointTable::HasUserCreatedViewpoint(const DBHandle& db) { + if (db->Exists(kHasUserCreatedViewpointKey)) { + return true; + } + for (DB::PrefixIterator iter(db, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const int64_t vp_id = state_->viewpoint_table()->DecodeContentKey(iter.key()); + ViewpointHandle vh = state_->viewpoint_table()->LoadViewpoint(vp_id, db); + if (!vh.get() || + vh->type() == "default" || + vh->type() == "system" || + vh->user_id() != state_->user_id()) { + continue; + } + db->Put(kHasUserCreatedViewpointKey, true); + return true; + } + return false; +} + +ViewpointTable::ContentHandle ViewpointTable::AddFollowers( + int64_t viewpoint_id, const vector& contacts) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + // Add followers. + vh->Lock(); + for (int i = 0; i < contacts.size(); ++i) { + if (contacts[i].has_user_id()) { + vh->AddFollower(contacts[i].user_id()); + } + } + vh->SaveAndUnlock(updates); + + // Create the local activity for adding followers. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + if (!AddContactsToAddFollowersActivity(ah, contacts)) { + updates->Abandon(); + return ContentHandle(); + } + ah->SaveAndUnlock(updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::RemoveFollowers( + int64_t viewpoint_id, const vector& user_ids) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + // Remove followers. + vh->Lock(); + for (int i = 0; i < user_ids.size(); ++i) { + vh->RemoveFollower(user_ids[i]); + } + vh->SaveAndUnlock(updates); + + // Create the local activity for removing followers. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + for (int i = 0; i < user_ids.size(); ++i) { + ah->mutable_remove_followers()->add_user_ids(user_ids[i]); + } + ah->SaveAndUnlock(updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::PostComment( + int64_t viewpoint_id, const string& message, + int64_t reply_to_photo_id) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + // Create the local activity for the comment. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + + // Create the comment. + CommentHandle ch = state_->comment_table()->NewComment(updates); + ch->Lock(); + ch->set_timestamp(timestamp); + ch->mutable_comment_id()->set_server_id( + EncodeCommentId(state_->device_id(), ch->comment_id().local_id(), + ch->timestamp())); + ch->mutable_viewpoint_id()->CopyFrom(vh->id()); + ch->set_user_id(state_->user_id()); + ch->set_message(message); + if (reply_to_photo_id != 0) { + PhotoHandle ph = state_->photo_table()->LoadPhoto(reply_to_photo_id, updates); + // TODO(spencer): if you're offline and add a photo to a + // conversation but the photo hasn't uploaded yet, this will skip + // the reply-to-photo. We might consider generating the server id + // earlier, instead of only at upload time. + if (ph.get() && !ph->id().server_id().empty()) { + ch->set_asset_id(ph->id().server_id()); + } + } + ch->SaveAndUnlock(updates); + + ah->mutable_post_comment()->mutable_comment_id()->CopyFrom(ch->comment_id()); + + ah->SaveAndUnlock(updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +void ViewpointTable::RemovePhotos( + const PhotoSelectionVec& photo_ids) { + DBHandle updates = state_->NewDBTransaction(); + + // For each photo id, query all episodes containing the photo that + // are part of the default viewpoint (e.g. visible in the library). + PhotoSelectionVec complete_ids; + for (int i = 0; i < photo_ids.size(); ++i) { + vector episode_ids; + if (state_->episode_table()->ListLibraryEpisodes( + photo_ids[i].photo_id, &episode_ids, updates)) { + for (int j = 0; j < episode_ids.size(); ++j) { + complete_ids.push_back(PhotoSelection(photo_ids[i].photo_id, episode_ids[j])); + } + } + } + + state_->episode_table()->RemovePhotos(complete_ids, updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); +} + +void ViewpointTable::SavePhotos( + const PhotoSelectionVec& photo_ids, int64_t autosave_viewpoint_id) { + if (!state_->is_registered()) { + return; + } + + DBHandle updates = state_->NewDBTransaction(); + SavePhotos(photo_ids, autosave_viewpoint_id, updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); +} + +void ViewpointTable::SavePhotos( + const PhotoSelectionVec& photo_ids, int64_t autosave_viewpoint_id, const DBHandle& updates) { + // Create the local activity for the save. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, updates); + if (!AddPhotosToSaveActivity( + state_, ah->mutable_save_photos()->mutable_episodes(), photo_ids, updates)) { + updates->Abandon(); + return; + } + if (autosave_viewpoint_id != 0) { + ViewpointHandle vh = LoadViewpoint(autosave_viewpoint_id, updates); + if (vh.get()) { + ah->mutable_save_photos()->mutable_viewpoint_id()->CopyFrom(vh->id()); + } + } + + ah->SaveAndUnlock(updates); +} + +ViewpointTable::ContentHandle ViewpointTable::ShareExisting( + int64_t viewpoint_id, const PhotoSelectionVec& photo_ids, + bool update_cover_photo) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + DCHECK(!vh->provisional()); + + // Create the local activity for the share. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + if (!AddPhotosToShareActivity( + state_, ah->mutable_share_existing()->mutable_episodes(), + vh, photo_ids, updates)) { + updates->Abandon(); + return ContentHandle(); + } + ah->SaveAndUnlock(updates); + state_->SetupViewpointTransition(viewpoint_id, updates); + + // Reset the cover photo if one isn't set or an update was specified. + if (!vh->has_cover_photo() || update_cover_photo) { + const ActivityMetadata::Episode* cover_episode = + ah->share_existing().episodes_size() > 0 ? + &ah->share_existing().episodes(0) : NULL; + if (cover_episode && cover_episode->photo_ids_size() > 0) { + vh->Lock(); + if (update_cover_photo) { + vh->set_update_metadata(true); + } + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom( + cover_episode->photo_ids(0)); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom( + cover_episode->episode_id()); + vh->SaveAndUnlock(updates); + } + } + + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::ShareNew( + const PhotoSelectionVec& photo_ids, + const vector& contacts, + const string& title, bool provisional) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + // Create the viewpoint for the share. + ViewpointHandle vh = NewViewpoint(updates); + vh->Lock(); + vh->mutable_id()->set_server_id( + EncodeViewpointId(state_->device_id(), vh->id().local_id())); + vh->set_user_id(state_->user_id()); + if (!title.empty()) { + vh->set_title(title); + } + if (provisional) { + vh->set_provisional(provisional); + } + // Set default labels admin and contribute. + vh->set_label_admin(true); + vh->set_label_contribute(true); + + // Add followers. + bool has_user_id = false; + vh->AddFollower(state_->user_id()); + for (int i = 0; i < contacts.size(); ++i) { + if (contacts[i].has_user_id()) { + if (contacts[i].user_id() == state_->user_id()) { + has_user_id = true; + } + vh->AddFollower(contacts[i].user_id()); + } + } + if (!has_user_id) { + vh->AddFollower(state_->user_id()); + } + vh->SaveAndUnlock(updates); + + // Create the local activity for the share. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + if (!AddPhotosToShareActivity( + state_, ah->mutable_share_new()->mutable_episodes(), + vh, photo_ids, updates)) { + updates->Abandon(); + return ContentHandle(); + } + if (!AddContactsToShareNewActivity(ah, contacts)) { + updates->Abandon(); + return ContentHandle(); + } + if (!has_user_id) { + ContactMetadata c; + if (state_->contact_manager()->LookupUser(state_->user_id(), &c)) { + ah->mutable_share_new()->add_contacts()->CopyFrom(c); + } + } + if (provisional) { + ah->set_provisional(provisional); + } + ah->SaveAndUnlock(updates); + + // Set the cover photo automatically to first shared photo. + const ActivityMetadata::Episode* cover_episode = + ah->share_new().episodes_size() > 0 ? + &ah->share_new().episodes(0) : NULL; + if (cover_episode && cover_episode->photo_ids_size() > 0) { + vh->Lock(); + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom( + cover_episode->photo_ids(0)); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom( + cover_episode->episode_id()); + vh->SaveAndUnlock(updates); + } + + state_->SetupViewpointTransition(vh->id().local_id(), updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +bool ViewpointTable::CommitShareNew(int64_t viewpoint_id, const DBHandle& updates) { + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return false; + } + vh->Lock(); + vh->clear_provisional(); + + // Mark any provisional activities as ready. + for (ScopedPtr iter( + state_->activity_table()->NewViewpointActivityIterator( + vh->id().local_id(), 0, false, updates)); + !iter->done(); + iter->Next()) { + ActivityHandle ah = iter->GetActivity(); + if (!ah.get() || !ah->provisional()) { + continue; + } + ah->Lock(); + ah->clear_provisional(); + if (ah->has_share_new()) { + // Update activity timestamp. + ah->set_timestamp(state_->WallTime_Now()); + + // Add followers which have user ids already. Prospective contacts + // will only be added as followers when the server reports back with + // a followers invalidation. + for (int i = 0; i < ah->share_new().contacts_size(); ++i) { + const ContactMetadata& c = ah->share_new().contacts(i); + if (c.has_user_id()) { + vh->AddFollower(c.user_id()); + } + } + } + ah->SaveAndUnlock(updates); + } + + vh->SaveAndUnlock(updates); + return true; +} + +bool ViewpointTable::UpdateShareNew( + int64_t viewpoint_id, int64_t activity_id, + const PhotoSelectionVec& photo_ids) { + if (!state_->is_registered()) { + return false; + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get() || !vh->provisional()) { + return false; + } + + ActivityHandle ah = state_->activity_table()->LoadActivity( + activity_id, updates); + if (!ah.get() || !ah->provisional()) { + return false; + } + + ah->Lock(); + ah->FilterShare(PhotoSelectionSet(photo_ids.begin(), photo_ids.end()), updates); + + if (!AddPhotosToShareActivity( + state_, ah->mutable_share_new()->mutable_episodes(), + vh, photo_ids, updates)) { + updates->Abandon(); + return false; + } + + // Update the timestamp in case there was a lag. Note that the + // original server id would have been encoded with a different + // timestamp, so these two could become out of sync. The timestamp + // baked into the server id is not used for anything on the server + // or client--it's mostly an aide to debugging. It will remain the + // time at which the client thought it first created the activity. + ah->set_timestamp(state_->WallTime_Now()); + ah->SaveAndUnlock(updates); + + // Set the cover photo automatically to first shared photo. + const ActivityMetadata::Episode* cover_episode = + ah->share_new().episodes_size() > 0 ? + &ah->share_new().episodes(0) : NULL; + vh->Lock(); + vh->clear_cover_photo(); + if (cover_episode && cover_episode->photo_ids_size() > 0) { + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom( + cover_episode->photo_ids(0)); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom( + cover_episode->episode_id()); + } + vh->SaveAndUnlock(updates); + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return true; +} + +ViewpointTable::ContentHandle ViewpointTable::Unshare( + int64_t viewpoint_id, const PhotoSelectionVec& photo_ids) { + if (!state_->is_registered()) { + return ContentHandle(); + } + + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + vector existing_episodes; + vh->ListEpisodes(&existing_episodes); + std::unordered_map episode_map; + for (int i = 0; i < existing_episodes.size(); ++i) { + episode_map[existing_episodes[i]->id().local_id()] = existing_episodes[i]; + } + + // Create the local activity for the unshare. + const WallTime timestamp = state_->WallTime_Now(); + ActivityHandle ah = NewLocalActivity(state_, timestamp, vh, updates); + ShareEpisodes* share_episodes = ah->mutable_unshare()->mutable_episodes(); + vector episodes; + bool unshared_cover_photo = false; + + for (int i = 0; i < photo_ids.size(); ++i) { + const int64_t photo_id = photo_ids[i].photo_id; + const int64_t episode_id = photo_ids[i].episode_id; + + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_id, updates); + if (!ph.get()) { + // Unable to find the photo to unshare. + updates->Abandon(); + return ContentHandle(); + } + + ActivityMetadata::Episode* e = NULL; + EpisodeHandle eh; + for (int j = 0; j < episodes.size(); ++j) { + if (episodes[j]->id().local_id() == episode_id) { + eh = episodes[j]; + e = share_episodes->Mutable(j); + break; + } + } + if (!e) { + if (!ContainsKey(episode_map, episode_id)) { + // Unable to find episode being unshared from. + updates->Abandon(); + return ContentHandle(); + } + eh = episode_map[episode_id]; + episodes.push_back(eh); + e = share_episodes->Add(); + e->mutable_episode_id()->CopyFrom(eh->id()); + eh->Lock(); + } + + e->add_photo_ids()->CopyFrom(ph->id()); + // Actually unshare the photo from the local episode. This will + // happen again when the notification for the unshare is received, + // but this updates the UI immediately. + eh->UnsharePhoto(ph->id().local_id()); + + if (vh->has_cover_photo() && + vh->cover_photo().photo_id().server_id() == ph->id().server_id() && + vh->cover_photo().episode_id().server_id() == eh->id().server_id()) { + unshared_cover_photo = true; + } + } + + if (share_episodes->size() == 0) { + ah->Unlock(); + updates->Abandon(); + return vh; + } + + // Save all episodes which have had photos unshared. + for (int i = 0; i < episodes.size(); ++i) { + episodes[i]->SaveAndUnlock(updates); + } + + ah->SaveAndUnlock(updates); + + // If we're unsharing the cover photo, set a new one locally. + if (unshared_cover_photo) { + ResetCoverPhoto(vh, updates); + } + + updates->Commit(); + + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::Remove( + int64_t viewpoint_id) { + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + if (!vh->label_removed()) { + vh->Lock(); + vh->set_label_removed(true); + vh->set_update_remove(true); + vh->SaveAndUnlock(updates); + + vh->InvalidateEpisodes(updates); + updates->Commit(); + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + } + + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateCoverPhoto( + int64_t viewpoint_id, int64_t photo_id, int64_t episode_id) { + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + PhotoHandle ph = state_->photo_table()->LoadPhoto(photo_id, updates); + EpisodeHandle eh = state_->episode_table()->LoadEpisode(episode_id, updates); + if (ph.get() && eh.get()) { + vh->Lock(); + vh->set_update_metadata(true); + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom(ph->id()); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom(eh->id()); + vh->SaveAndUnlock(updates); + updates->Commit(); + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + } + + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateTitle( + int64_t viewpoint_id, const string& title) { + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + vh->Lock(); + vh->set_title(title); + vh->set_update_metadata(true); + vh->SaveAndUnlock(updates); + updates->Commit(); + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + + return vh; +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateViewedSeq( + int64_t viewpoint_id) { + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + // Do not update the viewed sequence for provisional viewpoints. They are + // local to the client. + if (!vh->provisional() && vh->viewed_seq() < vh->update_seq()) { + vh->Lock(); + vh->set_viewed_seq(vh->update_seq()); + vh->set_update_viewed_seq(true); + vh->SaveAndUnlock(updates); + updates->Commit(); + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + } + + return vh; +} + +void ViewpointTable::SetScrollOffset(int64_t viewpoint_id, float offset) { + state_->db()->Put(EncodeViewpointScrollOffsetKey(viewpoint_id), offset); +} + +float ViewpointTable::GetScrollOffset(int64_t viewpoint_id) { + return state_->db()->Get(EncodeViewpointScrollOffsetKey(viewpoint_id), 0); +} + +bool ViewpointTable::ResetCoverPhoto( + const ViewpointHandle& vh, const DBHandle& updates) { + PhotoHandle ph; + EpisodeHandle eh = vh->GetAnchorEpisode(&ph); + vh->Lock(); + if (ph.get()) { + vh->mutable_cover_photo()->mutable_photo_id()->CopyFrom(ph->id()); + vh->mutable_cover_photo()->mutable_episode_id()->CopyFrom(eh->id()); + } else { + vh->clear_cover_photo(); + } + vh->SaveAndUnlock(updates); + return ph.get(); +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateAutosaveLabel( + int64_t viewpoint_id, bool autosave) { + return UpdateLabel(viewpoint_id, + &ViewpointMetadata::label_autosave, + &ViewpointMetadata::set_label_autosave, + &ViewpointMetadata::clear_label_autosave, + autosave); +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateHiddenLabel( + int64_t viewpoint_id, bool hidden) { + return UpdateLabel(viewpoint_id, + &ViewpointMetadata::label_hidden, + &ViewpointMetadata::set_label_hidden, + &ViewpointMetadata::clear_label_hidden, + hidden); +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateMutedLabel( + int64_t viewpoint_id, bool muted) { + return UpdateLabel(viewpoint_id, + &ViewpointMetadata::label_muted, + &ViewpointMetadata::set_label_muted, + &ViewpointMetadata::clear_label_muted, + muted); +} + +ViewpointTable::ContentHandle ViewpointTable::UpdateLabel( + int64_t viewpoint_id, + bool (ViewpointMetadata::*getter)() const, + void (ViewpointMetadata::*setter)(bool), + void (ViewpointMetadata::*clearer)(), + bool set_label) { + DBHandle updates = state_->NewDBTransaction(); + + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (!vh.get()) { + return ContentHandle(); + } + + if ((*vh.*getter)() != set_label) { + vh->Lock(); + if (set_label) { + (*vh.*setter)(true); + } else { + (*vh.*clearer)(); + } + vh->set_update_follower_metadata(true); + vh->SaveAndUnlock(updates); + vh->InvalidateEpisodes(updates); + updates->Commit(); + state_->async()->dispatch_after_main(0, [this] { + state_->net_manager()->Dispatch(); + }); + } + + return vh; +} + +void ViewpointTable::Validate( + const ViewpointSelection& s, const DBHandle& updates) { + const string key(DBFormat::viewpoint_selection_key(s.viewpoint_id())); + + // Load any existing viewpoint selection and clear attributes which have been + // queried by "s". If no attributes remain set, the selection is deleted. + ViewpointSelection existing; + if (updates->GetProto(key, &existing)) { + if (s.get_attributes()) { + existing.clear_get_attributes(); + } + if (s.get_followers()) { + if (!existing.get_followers() || + s.follower_start_key() <= existing.follower_start_key()) { + existing.clear_get_followers(); + existing.clear_follower_start_key(); + } + } else if (existing.get_followers()) { + existing.set_follower_start_key( + std::max(existing.follower_start_key(), + s.follower_start_key())); + } + if (s.get_activities()) { + if (!existing.get_activities() || + s.activity_start_key() <= existing.activity_start_key()) { + existing.clear_get_activities(); + existing.clear_activity_start_key(); + } + } else if (existing.get_activities()) { + existing.set_activity_start_key( + std::max(existing.activity_start_key(), + s.activity_start_key())); + } + if (s.get_episodes()) { + if (!existing.get_episodes() || + s.episode_start_key() <= existing.episode_start_key()) { + existing.clear_get_episodes(); + existing.clear_episode_start_key(); + } + } else if (existing.get_episodes()) { + existing.set_episode_start_key( + std::max(existing.episode_start_key(), + s.episode_start_key())); + } + if (s.get_comments()) { + if (!existing.get_comments() || + s.comment_start_key() <= existing.comment_start_key()) { + existing.clear_get_comments(); + existing.clear_comment_start_key(); + } + } else if (existing.get_comments()) { + existing.set_comment_start_key( + std::max(existing.comment_start_key(), + s.comment_start_key())); + } + } + + if (existing.has_get_attributes() || + existing.has_get_followers() || + existing.has_get_activities() || + existing.has_get_episodes() || + existing.has_get_comments()) { + updates->PutProto(key, existing); + } else { + updates->Delete(key); + } +} + +void ViewpointTable::Invalidate( + const ViewpointSelection& s, const DBHandle& updates) { + const string key(DBFormat::viewpoint_selection_key(s.viewpoint_id())); + + // Load any existing viewpoint selection and merge invalidations from "s". + ViewpointSelection existing; + if (!updates->GetProto(key, &existing)) { + existing.set_viewpoint_id(s.viewpoint_id()); + } + + if (s.get_attributes()) { + existing.set_get_attributes(true); + } + if (s.get_followers()) { + if (existing.get_followers()) { + existing.set_follower_start_key(std::min(existing.follower_start_key(), + s.follower_start_key())); + } else { + existing.set_follower_start_key(s.follower_start_key()); + } + existing.set_get_followers(true); + } + if (s.get_activities()) { + if (existing.get_activities()) { + existing.set_activity_start_key(std::min(existing.activity_start_key(), + s.activity_start_key())); + } else { + existing.set_activity_start_key(s.activity_start_key()); + } + existing.set_get_activities(true); + } + if (s.get_episodes()) { + if (existing.get_episodes()) { + existing.set_episode_start_key(std::min(existing.episode_start_key(), + s.episode_start_key())); + } else { + existing.set_episode_start_key(s.episode_start_key()); + } + existing.set_get_episodes(true); + } + if (s.get_comments()) { + if (existing.get_comments()) { + existing.set_comment_start_key(std::min(existing.comment_start_key(), + s.comment_start_key())); + } else { + existing.set_comment_start_key(s.comment_start_key()); + } + existing.set_get_comments(true); + } + + updates->PutProto(key, existing); +} + +void ViewpointTable::InvalidateFull( + const string& server_id, const DBHandle& updates) { + ViewpointSelection s; + s.set_viewpoint_id(server_id); + s.set_get_attributes(true); + s.set_get_activities(true); + s.set_get_episodes(true); + s.set_get_followers(true); + s.set_get_comments(true); + Invalidate(s, updates); +} + +void ViewpointTable::ListInvalidations( + vector* v, int limit, const DBHandle& db) { + v->clear(); + ScopedPtr iter(db->NewIterator()); + iter->Seek(kViewpointSelectionKeyPrefix); + while (iter->Valid() && (limit <= 0 || v->size() < limit)) { + Slice key = ToSlice(iter->key()); + if (!key.starts_with(kViewpointSelectionKeyPrefix)) { + break; + } + ViewpointSelection vps; + if (db->GetProto(key, &vps)) { + if (vps.has_viewpoint_id() && !vps.viewpoint_id().empty()) { + v->push_back(vps); + } else { + LOG("empty viewpoint id in viewpoint selection: %s; deleting", key); + state_->db()->Delete(key); + } + } else { + LOG("unable to read viewpoint selection at key %s; deleting", key); + state_->db()->Delete(key); + } + iter->Next(); + } +} + +void ViewpointTable::ClearAllInvalidations(const DBHandle& updates) { + ScopedPtr iter(updates->NewIterator()); + iter->Seek(kViewpointSelectionKeyPrefix); + for (; iter->Valid(); iter->Next()) { + Slice key = ToSlice(iter->key()); + if (!key.starts_with(kViewpointSelectionKeyPrefix)) { + break; + } + updates->Delete(key); + } +} + +void ViewpointTable::ProcessGCQueue() { + LOG("processing viewpoint garbage collection queue"); + DBHandle updates = state_->NewDBTransaction(); + for (DB::PrefixIterator iter(updates, kViewpointGCKeyPrefix); + iter.Valid(); + iter.Next()) { + int64_t viewpoint_id; + WallTime expiration; + if (DecodeViewpointGCKey(iter.key(), &viewpoint_id, &expiration)) { + if (state_->WallTime_Now() >= expiration) { + ViewpointHandle vh = LoadViewpoint(viewpoint_id, updates); + if (vh.get()) { + LOG("deleting viewpoint %s", vh->id()); + vh->Lock(); + vh->DeleteAndUnlock(updates); + } + updates->Delete(iter.key()); + } + } + } + updates->Commit(); +} + +bool ViewpointTable::FSCKImpl(int prev_fsck_version, const DBHandle& updates) { + LOG("FSCK: ViewpointTable"); + bool changes = false; + + for (DB::PrefixIterator iter(updates, DBFormat::viewpoint_key()); + iter.Valid(); + iter.Next()) { + const Slice key = iter.key(); + const Slice value = iter.value(); + ViewpointMetadata vm; + if (vm.ParseFromArray(value.data(), value.size())) { + ViewpointHandle vh = LoadViewpoint(vm.id().local_id(), updates); + vh->Lock(); + bool save_vh = false; + if (key != EncodeContentKey(DBFormat::viewpoint_key(), vm.id().local_id())) { + LOG("FSCK: viewpoint id %d does not equal key %s; deleting key and re-saving", + vm.id().local_id(), key); + updates->Delete(key); + save_vh = true; + } + + // Check server key mapping. This is special as viewpoints get created + // ahead of time when the first activity is received for a viewpoint + // which was shared with the user. We've experienced corruption issues + // in the past where an activity incorrectly refers to a viewpoint with + // no data via its local id while the "real" viewpoint arrives later and + // is assigned a subsequent local id and overrides the mapping. + // + // This code depends on the code in ActivityTable::FSCK rewriting any + // references as necessary. + if (vh->id().has_server_id()) { + const string server_key = EncodeContentServerKey(DBFormat::viewpoint_server_key(), + vh->id().server_id()); + if (!updates->Exists(server_key)) { + LOG("FSCK: missing viewpoint server key mapping"); + save_vh = true; + } else { + const int64_t mapped_local_id = updates->Get(server_key, -1); + if (mapped_local_id != vh->id().local_id()) { + LOG("FSCK: viewpoint local id mismatch: %d != %d; deleting existing mapping", + mapped_local_id, vh->id().local_id()); + updates->Delete(server_key); + ViewpointHandle mapped_vh = LoadViewpoint(mapped_local_id, updates); + if (mapped_vh.get()) { + LOG("FSCK: deleting incorrectly mapped viewpoint %s", *mapped_vh); + // List all followers, add to "canonical" viewpoint with lowest + // id and delete mappings to this vestigial viewpoint. + vector follower_ids; + mapped_vh->ListFollowers(&follower_ids); + for (int i = 0; i < follower_ids.size(); ++i) { + vh->AddFollower(follower_ids[i]); + updates->Delete(EncodeViewpointFollowerKey(mapped_vh->id().local_id(), follower_ids[i])); + updates->Delete(EncodeFollowerViewpointKey(follower_ids[i], mapped_vh->id().local_id())); + } + updates->Delete(EncodeContentKey(DBFormat::viewpoint_key(), mapped_local_id)); + } + save_vh = true; + } + } + } + + // Check required fields. + if (!vh->has_id() || + !vh->has_user_id()) { + LOG("FSCK: viewpoint missing required fields: %s", *vh); + if (vh->id().has_server_id() && !vh->id().server_id().empty()) { + LOG("FSCK: setting invalidation for viewpoint %s", *vh); + InvalidateFull(vh->id().server_id(), updates); + changes = true; + } + } + + if (save_vh) { + LOG("FSCK: rewriting viewpoint %s", *vh); + vh->SaveAndUnlock(updates); + changes = true; + } else { + vh->Unlock(); + } + } + } + + return changes; +} + +void ViewpointTable::Search(const Slice& query, ViewpointSearchResults* results) { + ScopedPtr parsed_query(FullTextQuery::Parse(query)); + for (ScopedPtr iter(viewpoint_index_->Search(state_->db(), *parsed_query)); + iter->Valid(); + iter->Next()) { + results->push_back(FastParseInt64(iter->doc_id())); + } +} + +string ViewpointTable::FormatViewpointToken(int64_t vp_id) { + return Format("%s%d_", kViewpointTokenPrefix, vp_id); +} + +void ViewpointTable::SaveContentHook(Viewpoint* viewpoint, const DBHandle& updates) { + vector terms; + int pos = viewpoint_index_->ParseIndexTerms(0, viewpoint->title(), &terms); + + vector followers; + viewpoint->ListFollowers(&followers); + for (auto user_id : followers) { + pos = viewpoint_index_->AddVerbatimToken(pos, ContactManager::FormatUserToken(user_id), &terms); + } + + if (viewpoint->label_removed()) { + // If the viewpoint has been removed, clear out its terms so it won't show + // up in the autocomplete. + // It would be nice to do the same for comments, but that's more expensive + // and the comments are not as individually prominent in the results. + terms.clear(); + } + + // TODO(ben): Ensure that the viewpoint is re-saved when an activity is added. + ActivityHandle ah = state_->activity_table()->GetLatestActivity(viewpoint->id().local_id(), updates); + const string sort_key = ah.get() ? FullTextIndex::TimestampSortKey(ah->timestamp()) : ""; + viewpoint_index_->UpdateIndex(terms, ToString(viewpoint->id().local_id()), sort_key, + viewpoint->mutable_indexed_terms(), updates); +} + +void ViewpointTable::DeleteContentHook(Viewpoint* viewpoint, const DBHandle& updates) { + viewpoint_index_->RemoveTerms(viewpoint->mutable_indexed_terms(), updates); +} + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/ViewpointTable.h b/clients/shared/ViewpointTable.h new file mode 100644 index 0000000..3dc2de4 --- /dev/null +++ b/clients/shared/ViewpointTable.h @@ -0,0 +1,339 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_VIEWPOINT_TABLE_H +#define VIEWFINDER_VIEWPOINT_TABLE_H + +#import "ContentTable.h" +#import "DB.h" +#import "EpisodeTable.h" +#import "InvalidateMetadata.pb.h" +#import "PhotoSelection.h" +#import "ScopedHandle.h" +#import "ViewpointMetadata.pb.h" + +class ActivityTable; +class AppState; +class ContactMetadata; +class FullTextIndex; +class NetworkQueue; +class PhotoTable; + +// The ViewpointTable class maintains the mappings: +// -> +// -> +// -> +// , -> <> (follower table) +// -> +// , -> <> (reverse follower table) +// +// ViewpointTable is thread-safe and ViewpointHandle is thread-safe, but individual +// Viewpoints are not. + +class ViewpointTable_Viewpoint : public ViewpointMetadata { + enum FollowerState { + // It is important for ADDED to be have the value 0 so that that value is + // the default state for a follower. + ADDED = 0, + LOADED, + REMOVED, + }; + + typedef std::map FollowerStateMap; + + public: + static const string kTypeDefault; + static const double kViewpointGCExpirationSeconds; + + public: + virtual void MergeFrom(const ViewpointMetadata& m); + // Unimplemented; exists to get the compiler not to complain about hiding the base class's overloaded MergeFrom. + virtual void MergeFrom(const ::google::protobuf::Message&); + + // Adds a follower to the viewpoint. The follower is persisted when + // Save() is called. + void AddFollower(int64_t follower_id); + + // Removes a follower from the viewpoint. + void RemoveFollower(int64_t follower_id); + + // Returns a count of the number of followers. + int CountFollowers(); + + // Lists the followers of the viewpoint. Note that this will not + // include prospective users which have not yet been sent to the + // server (e.g. that don't yet have a user id). + void ListFollowers(vector* follower_ids); + + // List the follower ids removable by the current user. This is + // always true for the user himself. It is true otherwise if the + // user added the specified follower in the past 7 days. + void GetRemovableFollowers(std::unordered_set* removable); + + // List the episodes that are associated with the viewpoint. + void ListEpisodes(vector* episodes); + + // Returns the first sharing episode with at least one valid photo. + // If "ph_ptr" is not NULL, it is set to the anchor episode's first + // valid photo. + EpisodeHandle GetAnchorEpisode(PhotoHandle* ph_ptr); + + // Returns the photo id and aspect ratio of this viewpoint's cover photo. + // If not set explicitly in the viewpoint metadata, a cover photo is + // chosen automatically. If no photos are available, returns false. + bool GetCoverPhoto(int64_t* photo_id, int64_t* episode_id, + WallTime* timestamp, float* aspect_ratio); + + // Returns the viewpoint title. If none has been set explicitly, + // creates one based on the content of the viewpoint. + string FormatTitle(bool shorten, bool normalize_whitespace = false); + + // Returns the default title for use when no title has been explicitly + // set for the viewpoint. + string DefaultTitle(); + + // Invalidate all episodes in the viewpoint. Called when personal label + // is set or viewpoint is removed--will properly update all event trapdoors. + void InvalidateEpisodes(const DBHandle& updates); + + bool is_default() const { return type() == kTypeDefault; } + + // Returns the expiration wall time for garbage collecting this viewpoint. + // If the viewpoint has been marked "unrevivable", expiration is immediate; + // otherwise, expiration is +kViewpointGCExpirationSeconds from current time. + float GetGCExpiration(); + + protected: + // Store the original cover photo information if available in order to + // notice changes and invalidate appropriately. + bool Load(); + // If the cover photo has changed, invalidates both the previous + // cover photo's episode (if one was set) and the new cover photo's + // episode. + void SaveHook(const DBHandle& updates); + void DeleteHook(const DBHandle& updates); + + string GetDayTableFields() const; + + int64_t local_id() const { return id().local_id(); } + const string& server_id() const { return id().server_id(); } + + ViewpointTable_Viewpoint(AppState* state, const DBHandle& db, int64_t id); + + protected: + AppState* state_; + DBHandle db_; + + private: + void EnsureFollowerState(); + // Adds metadata for user_id to users if it is not already present in unique_users. + void AddUniqueUser(int64_t user_id, std::unordered_set* unique_users, vector* users); + + private: + // A cache of the followers, loaded on demand. + ScopedPtr followers_; + CoverPhoto orig_cover_photo_; + bool disk_label_removed_; + string day_table_fields_; +}; + +class ViewpointTable : public ContentTable { + typedef ViewpointTable_Viewpoint Viewpoint; + + public: + ViewpointTable(AppState* state); + ~ViewpointTable(); + + ContentHandle NewViewpoint(const DBHandle& updates) { + return NewContent(updates); + } + ContentHandle LoadViewpoint(int64_t id, const DBHandle& db) { + return LoadContent(id, db); + } + ContentHandle LoadViewpoint(const string& server_id, const DBHandle& db) { + return LoadContent(server_id, db); + } + ContentHandle LoadViewpoint(const ViewpointId& id, const DBHandle& db); + + // Looks up the viewpoint by "vp_id->server_id" if available and + // sets "vp_id->local_id". If the viewpoint doesn't exist, a local, + // empty viewpoint is created. + void CanonicalizeViewpointId(ViewpointId* vp_id, const DBHandle& updates); + + // Lists the viewpoints (other than default) the specified photo has + // been shared to. + void ListViewpointsForPhotoId( + int64_t photo_id, vector* viewpoint_ids, const DBHandle& db); + + // Lists the viewpoints the specified user is a follower of. + void ListViewpointsForUserId( + int64_t user_id, vector* viewpoint_ids, const DBHandle& db); + + // Returns whether "self" user has created a viewpoint. + bool HasUserCreatedViewpoint(const DBHandle& db); + + // Add followers to an existing viewpoint. + ContentHandle AddFollowers( + int64_t viewpoint_id, const vector& contacts); + + // Remove followers from an existing viewpoint. + ContentHandle RemoveFollowers( + int64_t viewpoint_id, const vector& user_ids); + + // Posts a comment to an existing viewpoint. If "reply_to_photo_id" is + // not 0, sets the "asset_id" in the post metadata to the server id + // of the photo in question. + ContentHandle PostComment( + int64_t viewpoint_id, const string& message, int64_t reply_to_photo_id); + + // Removes photos from all saved library episodes. The episode id listed + // with each photo in the selection vector is ignored. Instead, uses + // EpisodeTable::ListLibraryEpisodes to get the complete list of saved + // episodes. + void RemovePhotos(const PhotoSelectionVec& photo_ids); + + // Saves photos from conversation episodes to library episodes which are + // part of the default viewpoint. Specify autosave_viewpoint_id to indicate + // all photos in the viewpoint should be saved. This handles a possible + // race condition where new photos are added between the client gathering + // photo ids ("photo_ids") and this call going through to the server. The + // server will handle adding any additional photos when the operation is + // executed. Specify autosave_viewpoint_id=0 otherwise. + void SavePhotos(const PhotoSelectionVec& photo_ids, int64_t autosave_viewpoint_id); + void SavePhotos(const PhotoSelectionVec& photo_ids, + int64_t autosave_viewpoint_id, const DBHandle& updates); + + // Shares photos to an existing viewpoint. Returns NULL on failure + // and the viewpoint on success. If "update_cover_photo" is + // specified, the cover photo will be modified to the first photo + // in the photo_ids selection vec. + ContentHandle ShareExisting( + int64_t viewpoint_id, const PhotoSelectionVec& photo_ids, + bool update_cover_photo); + + // Shares photos to a new viewpoint. Returns NULL on failure and the new + // viewpoint on success. If "provisional" is true, the new viewpoint will not + // be uploaded to the server until the provisional bit is cleared. + ContentHandle ShareNew( + const PhotoSelectionVec& photo_ids, + const vector& contacts, + const string& title, bool provisional); + + // Commit a provisional viewpoint, allowing it to be uploaded to the server. + bool CommitShareNew(int64_t viewpoint_id, const DBHandle& updates); + + // Update an existing share new activity. Returns false if the activity could + // not be updated (e.g it is not provisional or does not exist) and true if + // the activity was updated. Note that any existing photos in the share new + // activity are replaced with the photos specified in the photo_ids vector. + // The activity's timestamp is updated to the current time. + bool UpdateShareNew( + int64_t viewpoint_id, int64_t activity_id, + const PhotoSelectionVec& photo_ids); + + // Unshares photos from an existing viewpoint. Returns NULL on failure and + // the viewpoint on success. + ContentHandle Unshare( + int64_t viewpoint_id, const PhotoSelectionVec& photo_ids); + + // Removes the viewpoint from the inbox view if label_removed has + // not been set. This invokes /service/remove_viewpoint on the server. + ContentHandle Remove(int64_t viewpoint_id); + + // Updates the viewpoint cover photo. + ContentHandle UpdateCoverPhoto(int64_t viewpoint_id, int64_t photo_id, int64_t episode_id); + + // Updates the viewpoint title. + ContentHandle UpdateTitle(int64_t viewpoint_id, const string& title); + + // Updates viewpoint "viewed_seq" property on the server to mark + // the viewpoint as viewed. + ContentHandle UpdateViewedSeq(int64_t viewpoint_id); + + // Sets viewpoint "scroll_offset" property. + void SetScrollOffset(int64_t viewpoint_id, float offset); + + // Returns viewpoint "scroll_offset" property. + float GetScrollOffset(int64_t viewpoint_id); + + // Reset the cover photo based on the first available share activity + // (photo neither removed nor unshared). + bool ResetCoverPhoto(const ContentHandle& vh, const DBHandle& updates); + + // Update viewpoint labels. + ContentHandle UpdateAutosaveLabel(int64_t viewpoint_id, bool autosave); + ContentHandle UpdateHiddenLabel(int64_t viewpoint_id, bool hidden); + ContentHandle UpdateMutedLabel(int64_t viewpoint_id, bool muted); + + // TODO(spencer): Move the validation/invalidation code into ContentTable template. + + // Validates portions of the specified viewpoint. + // IMPORTANT: this method must be kept in sync with the current set of + // attributes contained in ViewpointSelection. + void Validate(const ViewpointSelection& s, const DBHandle& updates); + + // Invalidates portions of the specified viewpoint. + // IMPORTANT: this method must be kept in sync with the current set of + // attributes contained in ViewpointSelection. + void Invalidate(const ViewpointSelection& s, const DBHandle& updates); + + // Fully invalidates the viewpoint specified by "server_id". + void InvalidateFull(const string& server_id, const DBHandle& updates); + + // Lists the viewpoint invalidations. Specify limit=0 for all invalidations. + void ListInvalidations(vector* v, int limit, const DBHandle& db); + + // Clear all of the viewpoint invalidations. + void ClearAllInvalidations(const DBHandle& updates); + + // Process any pending viewpoints for garbage collection. When a + // viewpoint is garbage collected, its contents are recursively + // removed from the DB. + void ProcessGCQueue(); + + // Verifies there are no orphaned viewpoints, created to satisfy a + // CanonicalizeViewpointId call. + bool FSCKImpl(int prev_fsck_version, const DBHandle& updates); + + typedef vector ViewpointSearchResults; + void Search(const Slice& query, ViewpointSearchResults* results); + + FullTextIndex* viewpoint_index() const { return viewpoint_index_.get(); } + + // Returns a string that can be used in search (in other indexes) to find + // records related to the given viewpoint. + static string FormatViewpointToken(int64_t vp_id); + + // Prefix common to all FormatViewpointToken() strings. + static const string kViewpointTokenPrefix; + + protected: + virtual void SaveContentHook(Viewpoint* viewpoint, const DBHandle& updates); + virtual void DeleteContentHook(Viewpoint* viewpoint, const DBHandle& updates); + + private: + ContentHandle UpdateLabel(int64_t viewpoint_id, + bool (ViewpointMetadata::*getter)() const, + void (ViewpointMetadata::*setter)(bool), + void (ViewpointMetadata::*clearer)(), + bool set_label); + + ScopedPtr viewpoint_index_; +}; + +typedef ViewpointTable::ContentHandle ViewpointHandle; + +string EncodeFollowerViewpointKey(int64_t follower_id, int64_t viewpoint_id); +string EncodeViewpointFollowerKey(int64_t viewpoint_id, int64_t follower_id); +string EncodeViewpointScrollOffsetKey(int64_t viewpoint_id); +string EncodeViewpointGCKey(int64_t viewpont_id, WallTime expiration); +bool DecodeFollowerViewpointKey(Slice key, int64_t* follower_id, int64_t* viewpoint_id); +bool DecodeViewpointFollowerKey(Slice key, int64_t* viewpoint_id, int64_t* follower_id); +bool DecodeViewpointScrollOffsetKey(Slice key, int64_t* viewpoint_id); +bool DecodeViewpointGCKey(Slice key, int64_t* viewpoint_id, WallTime* expiration); + +#endif // VIEWFINDER_VIEWPOINT_TABLE_H + +// local variables: +// mode: c++ +// end: diff --git a/clients/shared/WallTime.android.cc b/clients/shared/WallTime.android.cc new file mode 100644 index 0000000..8816c12 --- /dev/null +++ b/clients/shared/WallTime.android.cc @@ -0,0 +1,24 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "WallTime.h" +#import "WallTime.android.h" + +std::function time_zone_offset; +std::function time_zone_name; + +struct tm LocalTime(WallTime time) { + // Timezones appear to be broken in Android JNI code. Specifically, the + // current timezone appears to always be either UTC or GMT with no ability to + // change via tzset(). Google reveals that localtime_r() is essentially just + // a small wrapper around gmtime_r(). + // + // NOTE(peter): Note that time_zone_offset results in a call into Java. I + // hope this is fast enough because making it faster will be painful. + const time_t time_sec = static_cast(time); + const time_t local_time_sec = time_sec - + (time_zone_offset ? time_zone_offset(time_sec) : 0); + struct tm t; + gmtime_r(&local_time_sec, &t); + return t; +} diff --git a/clients/shared/WallTime.android.h b/clients/shared/WallTime.android.h new file mode 100644 index 0000000..495677b --- /dev/null +++ b/clients/shared/WallTime.android.h @@ -0,0 +1,14 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_WALLTIME_ANDROID_H +#define VIEWFINDER_WALLTIME_ANDROID_H + +#import +#import +#import + +extern std::function time_zone_offset; +extern std::function time_zone_name; + +#endif // VIEWFINDER_WALLTIME_ANDROID_H diff --git a/clients/shared/WallTime.cc b/clients/shared/WallTime.cc new file mode 100644 index 0000000..8d43b4e --- /dev/null +++ b/clients/shared/WallTime.cc @@ -0,0 +1,286 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import +#import +#import "Logging.h" +#import "StringUtils.h" +#import "WallTime.h" + +namespace { + +const WallTime kMinute = 60; +const WallTime kHour = 60 * kMinute; +const WallTime kDay = 24 * kHour; +const WallTime kHalfDay = 12 * kHour; +const WallTime kWeek = 7 * kDay; +const WallTime kMonth = 30 * kDay; + +void PrintOne(ostream& os, struct tm& t, + const string& fmt, int begin, int end) { + if (begin != end) { + char buf[128]; + if ((begin == 0) && (end == fmt.size())) { + strftime(buf, sizeof(buf), fmt.c_str(), &t); + } else { + strftime(buf, sizeof(buf), fmt.substr(begin, end - begin).c_str(), &t); + } + os << buf; + } +} + +struct { + double max_seconds; + int divisor; + const char* fmt; + const char* medium_fmt; + const char* long_fmt; +} kTimeAgoRanges[] = { + { 1, 1, "now", "just now", "just now" }, + // Fractions are rounded, so the transition between singular and plural happens at 1.5x the base unit. + { 1.5, 1, "%.0fs", "%.0fs ago", "%.0f second ago" }, + { 60, 1, "%.0fs", "%.0fs ago", "%.0f seconds ago" }, + { 60 * 1.5, 60, "%.0fm", "%.0fm ago", "%.0f minute ago" }, + { 60 * 60, 60, "%.0fm", "%.0fm ago", "%.0f minutes ago" }, + { 60 * 60 * 1.5, 60 * 60, "%.0fh", "%.0fh ago", "%.0f hour ago" }, + { 60 * 60 * 24, 60 * 60, "%.0fh", "%.0fh ago", "%.0f hours ago" }, + { 60 * 60 * 24 * 1.5, 60 * 60 * 24, "%.0fd", "%.0fd ago", "%.0f day ago" }, + { 60 * 60 * 24 * 7, 60 * 60 * 24, "%.0fd", "%.0fd ago", "%.0f days ago" }, +}; + +} // namespace + +WallTime WallTime_Now() { + struct timeval t; + gettimeofday(&t, NULL); + return t.tv_sec + (t.tv_usec / 1e6); +} + +WallTime CurrentHour(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime CurrentDay(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime CurrentMonth(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday = 1; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime CurrentYear(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday = 1; + t.tm_mon = 0; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime NextHour(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour += 1; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime NextDay(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday += 1; + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime NextMonth(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday = 1; + if (t.tm_mon == 11) { + t.tm_year += 1; + t.tm_mon = 0; + } else { + t.tm_mon += 1; + } + t.tm_isdst = -1; + return mktime(&t); +} + +WallTime NextYear(WallTime time) { + struct tm t = LocalTime(time); + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday = 1; + t.tm_mon = 0; + t.tm_year += 1; + t.tm_isdst = -1; + return mktime(&t); +} + +string FormatTime(WallTime time) { + const WallTime cur_day = CurrentDay(time); + return Trim(Format("%s%s", WallTimeFormat("%l:%M", time), + (time - cur_day < kHalfDay ? "a" : "p"))); +} + +string FormatDate(const char* fmt, WallTime time) { + string date = Format("%s", WallTimeFormat(fmt, time)); + for (int pos = 0; (pos = date.find(" ", pos)) != string::npos; ) { + date.erase(pos, 1); + } + return date; +} + +string FormatRelativeTime(WallTime t, WallTime now) { + if (CurrentDay(t) == CurrentDay(now)) { + return FormatTime(t); + } + if (fabs(t - now) < kWeek) { + return Format("%s%s", WallTimeFormat("%a, ", t), FormatTime(t)); + } else if (CurrentYear(t) == CurrentYear(now)) { + return Format("%s%s", FormatDate("%b %e, ", t), FormatTime(t)); + } + return Format("%s%s", FormatDate("%b %e, %Y, ", t), FormatTime(t)); +} + +string FormatTimeAgo(WallTime ago, WallTime now, TimeAgoFormat format) { + double elapsed = now - ago; + if (elapsed < 1) { + return (format == TIME_AGO_SHORT) ? "now" : "just now"; + } + for (int i = 0; i < ARRAYSIZE(kTimeAgoRanges); ++i) { + if (elapsed < kTimeAgoRanges[i].max_seconds) { + const char* format_string; + if (format == TIME_AGO_SHORT) { + format_string = kTimeAgoRanges[i].fmt; + } else if (format == TIME_AGO_MEDIUM) { + format_string = kTimeAgoRanges[i].medium_fmt; + } else { + format_string = kTimeAgoRanges[i].long_fmt; + } + return Format(format_string, elapsed / kTimeAgoRanges[i].divisor); + } + } + if (CurrentYear(ago) != CurrentYear(now)) { + if (format == TIME_AGO_SHORT) { + return Format("%s", WallTimeFormat("%b %e, %Y", ago)); + } else { + return Format("%s", WallTimeFormat("on %b %e, %Y", ago)); + } + } else { + if (format == TIME_AGO_SHORT) { + return Format("%s", FormatDate("%b %e", ago)); + } else { + return Format("%s", FormatDate("on %b %e", ago)); + } + } +} + +string FormatTimeRange(WallTime begin, WallTime end) { + const bool within_12_hours = fabs(end - begin) < kDay / 2; + const bool same_day = int(end / kDay) == int(begin / kDay); + + if (within_12_hours || same_day) { + if (int(end / kMinute) == int(begin / kMinute)) { + return Format("%s, %s", WallTimeFormat("%a, %b %e, %Y", begin), FormatTime(begin)); + } else { + return Format("%s, %s \u2014 %s", WallTimeFormat("%a, %b %e, %Y", begin), + FormatTime(begin), FormatTime(end)); + } + } + return Format("%s, %s \u2014 %s, %s", + WallTimeFormat("%a, %b %e, %Y", begin), FormatTime(begin), + WallTimeFormat("%a, %b %e, %Y", end), FormatTime(end)); +} + +string FormatRelativeDate(WallTime d, WallTime now) { + if (CurrentYear(d) == CurrentYear(now)) { + return Format("%s", FormatDate("%a, %b %e", d)); + } + return Format("%s", FormatDate("%b %e, %Y", d)); +} + +string FormatShortRelativeDate(WallTime d, WallTime now) { + if (CurrentYear(d) == CurrentYear(now)) { + return Format("%s", FormatDate("%b %e", d)); + } else { + return Format("%s", WallTimeFormat("%b %e '%y", d)); + } +} + +string FormatDateRange(WallTime begin, WallTime end, WallTime now) { + bool same_day = fabs(end - begin) < kDay; + + if (same_day) { + return FormatRelativeDate(end, now); + } + return Format("%s \u2014 %s", FormatRelativeDate(begin, now), + FormatRelativeDate(end, now)); +} + +void WallTimeSleep(WallTime t) { + timespec ts; + ts.tv_sec = static_cast(t); + ts.tv_nsec = (t - ts.tv_sec) * 1e9; + nanosleep(&ts, NULL); +} + +void WallTimeFormat::Print(ostream& os) const { + const time_t time_sec = static_cast(time_); + struct tm t; + if (localtime_) { + localtime_r(&time_sec, &t); + } else { + gmtime_r(&time_sec, &t); + } + + int last = 0; + for (int i = 0; i < fmt_.size(); ) { + const int p = fmt_.find('%', i); + if (p == fmt_.npos) { + break; + } + if (p + 1 < fmt_.size()) { + if ((fmt_[p + 1] == 'Q') || (fmt_[p + 1] == 'N')) { + // One of our special formatting characters. Output everything up to the + // last + PrintOne(os, t, fmt_, last, p); + if (fmt_[p + 1] == 'Q') { + os << Format("%03d") % static_cast(1e3 * (time_ - time_sec)); + } else if (fmt_[p + 1] == 'N') { + os << Format("%06d") % static_cast(1e6 * (time_ - time_sec)); + } + i = p + 2; + last = i; + continue; + } + } + i = p + 1; + } + + PrintOne(os, t, fmt_, last, fmt_.size()); +} diff --git a/clients/shared/WallTime.h b/clients/shared/WallTime.h new file mode 100644 index 0000000..06bad47 --- /dev/null +++ b/clients/shared/WallTime.h @@ -0,0 +1,105 @@ +// Copyright 2012 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#ifndef VIEWFINDER_WALLTIME_H +#define VIEWFINDER_WALLTIME_H + +#import +#import +#import "Utils.h" + +typedef double WallTime; + +WallTime WallTime_Now(); +WallTime CurrentHour(WallTime t); +WallTime CurrentDay(WallTime t); +WallTime CurrentMonth(WallTime t); +WallTime CurrentYear(WallTime t); +WallTime NextHour(WallTime t); +WallTime NextDay(WallTime t); +WallTime NextMonth(WallTime t); +WallTime NextYear(WallTime t); +struct tm LocalTime(WallTime t); + +// Formats time with a trailing "a" or "p" for AM and PM respectively. +string FormatTime(WallTime time); + +// Remove the double-spaces which WallTimeFormat adds in the case of +// single-digit day of month value. +string FormatDate(const char* fmt, WallTime time); + +// Format a time (hours and minutes) relative to "now". Always includes +// the time, but depending on elapsed time since "t", may include +// progressively general date information as well. +string FormatRelativeTime(WallTime t, WallTime now); + +enum TimeAgoFormat { + TIME_AGO_SHORT, // e.g. "42m" + TIME_AGO_MEDIUM, // e.g. "42m ago" + TIME_AGO_LONG, // e.g. "42 minutes ago" +}; +// Format "time ago", a relative expression of time elapsed since the +// time "ago" to "now". +string FormatTimeAgo(WallTime ago, WallTime now, TimeAgoFormat format); +// Format a time range using abbreviated weekday, month and full +// 4-digit year. If the times are within the same day or if they cross +// days but are less than 12 hours apart, displays just the time +// differential. If the times cross days and are 12 hours or more +// apart, displays full date and time range using abbreviated weekday, +// month and full 4-digit year. +string FormatTimeRange(WallTime begin, WallTime end); + +// Format a date relative to "now". Time is included if the date is +// within the last week. Otherwise, depending on elapsed time since +// "t", includes progressively general date information. +string FormatRelativeDate(WallTime d, WallTime now); +// Formats a short version of the relative date. +string FormatShortRelativeDate(WallTime d, WallTime now); +// Format a date range relative to the current time. +string FormatDateRange(WallTime begin, WallTime end, WallTime now); + +// Sleep the current thread for the specified duration. +void WallTimeSleep(WallTime t); + +class WallTimeFormat { + public: + // Like strftime() format, but also accepts %Q for milliseconds and %N for + // nanonseconds. + WallTimeFormat(const string& fmt, WallTime time, bool localtime = true) + : fmt_(fmt), + time_(time), + localtime_(localtime) { + } + + void Print(ostream& os) const; + + private: + string fmt_; + const WallTime time_; + const bool localtime_; +}; + +struct WallTimeInterval { + public: + WallTimeInterval(WallTime b = 0, WallTime e = 0) + : begin(b), + end(e) { + } + + WallTime size() const { return end - begin; } + + WallTime begin; + WallTime end; +}; + +inline bool operator<(const WallTimeInterval& a, const WallTimeInterval& b) { + return a.begin < b.begin; +} + +inline ostream& operator<<( + ostream& os, const WallTimeFormat& f) { + f.Print(os); + return os; +} + +#endif // VIEWFINDER_WALLTIME_H diff --git a/clients/shared/WallTime.ios.cc b/clients/shared/WallTime.ios.cc new file mode 100644 index 0000000..e2faae2 --- /dev/null +++ b/clients/shared/WallTime.ios.cc @@ -0,0 +1,11 @@ +// Copyright 2013 Viewfinder. All rights reserved. +// Author: Peter Mattis. + +#import "WallTime.h" + +struct tm LocalTime(WallTime time) { + const time_t time_sec = static_cast(time); + struct tm t; + localtime_r(&time_sec, &t); + return t; +} diff --git a/clients/shared/protoc.gypi b/clients/shared/protoc.gypi new file mode 100644 index 0000000..30fd1fc --- /dev/null +++ b/clients/shared/protoc.gypi @@ -0,0 +1,61 @@ +# Based on http://src.chromium.org/svn/trunk/src/build/protoc.gypi +# +# Include this rule in a library or executable target to generate the +# c++ code for all .proto files listed in 'sources'. +{ + 'conditions': [ + ['OS=="ios"', { + 'variables': { + 'protoc_wrapper%': '<(DEPTH)/../shared/scripts/protoc-gyp.sh', + }, + 'all_dependent_settings': { + 'include_dirs': [ + '${SHARED_INTERMEDIATE_DIR}/protoc_out', + ], + }, + },], + ['OS=="android"', { + 'variables': { + 'protoc_wrapper%': '<(DEPTH)/clients/shared/scripts/protoc-gyp.sh', + }, + 'all_dependent_settings': { + 'include_dirs': [ + # HACK: on android $SHARED_INTERMEDIATE_DIR is not actually shared, and will be evaluated + # in the context of the dependent target. We must use an explicitly relative path here + # so it will be resolved correctly. + './protoc_out', + ], + }, + },], + ], + 'variables': { + 'proto_out_dir%': '', + }, + 'rules': [ + { + 'rule_name': 'genproto', + 'extension': 'proto', + 'inputs': [ + '<(protoc_wrapper)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/protoc_out/<(proto_out_dir)<(RULE_INPUT_ROOT).pb.cc', + '<(SHARED_INTERMEDIATE_DIR)/protoc_out/<(proto_out_dir)<(RULE_INPUT_ROOT).pb.h', + ], + 'action': [ + 'bash', + '<(protoc_wrapper)', + '>(_include_dirs)', + '<(RULE_INPUT_PATH)', + '<(SHARED_INTERMEDIATE_DIR)/protoc_out/<(proto_out_dir)' + ], + 'message': 'Generating C++ code from <(RULE_INPUT_PATH)', + 'process_outputs_as_sources': 1, + }, + ], + 'include_dirs': [ + '${SHARED_INTERMEDIATE_DIR}/protoc_out', + ], + # This target exports a hard dependency because it generates header files. + 'hard_dependency': 1, +} diff --git a/clients/shared/scripts/assets-tool.py b/clients/shared/scripts/assets-tool.py new file mode 100644 index 0000000..06234e4 --- /dev/null +++ b/clients/shared/scripts/assets-tool.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python +# +# Copyright 2013 Viefinder Inc. All Rights Reserved. + +__author__ = 'marc@emailscrubbed.com (Marc Berhault)' + +import glob +import itertools +import os +import re +import shutil + +from PIL import Image +from tornado import options + +options.define('ninepatch', default=False, help='Generate android 9patch assets') +options.define('plain', default=False, help='Verify existence of plain assets on android') +options.define('regenerate', type=str, default=[], help='Regenerate 9patch is different. Names are source iOS images') +options.define('v', default=False, help='Verbose') + +# List of files (name of iOS source image) to skip when processing plain assets. +SKIP_PLAIN = [] + +# List of files that are implicitly referenced. +FORCE_REFERENCE = ['Icon.png'] + +# This assumes the script is in ${VF_HOME}/clients/shared/scripts/ +BASE_PATH=os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../')) + +# Paths relative to viewfinder home. +IOS_SOURCE_DIR=os.path.join(BASE_PATH, 'clients/ios/Source') +IOS_IMAGES_DIR=os.path.join(BASE_PATH, 'clients/ios/Source/Images') +# For now, assume they all go into x +ANDROID_IMAGES_DIR=os.path.join(BASE_PATH, 'clients/android/res/drawable-xhdpi') + +def ImageName2X(name): + base, ext = name.rsplit('.', 1) + return '%s@2x.%s' % (base, ext) + +def ImageName9Patch(name): + base, ext = name.split('.') + return ImageNameAndroid('%s.9.%s' % (base, ext)) + +def ImageNameAndroid(name): + return name.replace('-', '_').lower() + +def FindFilesIn(directory, pattern): + """ Returns the list of all files in 'directory' matching the glob 'pattern'. """ + return glob.glob(os.path.join(directory, pattern)) + + +def ListBasename(filelist): + """ Turns a list of paths into a list of basenames for each entry. """ + return [os.path.basename(f) for f in filelist] + + +def FindReferencedIOSImages(): + """ Find images referenced in the code and return a dict of "filename" -> "inset quad". """ + source_files = FindFilesIn(IOS_SOURCE_DIR, '*.mm') + image_re = re.compile(r'^LazyStaticImage [^;]+;', re.MULTILINE) + file_re = re.compile(r'@"([^"]+)"') + # We're assuming the UIEdgeInsetsMake call is not split across lines. We strip whitespace away before searching. + inset_re = re.compile(r'UIEdgeInsetsMake\(([0-9.]+),([0-9.]+),([0-9.]+),([0-9.]+)\)') + + def ParseLazyStaticImage(line): + # Extract name + result = file_re.search(line) + assert result is not None, 'Could not extract filename from %r' % line + name = result.group(1) + # iOS assumes a .png extension if none is specified. + if '.' not in name: + name += '.png' + + # Look for edge inset. Replace whitespace to make the regexp simpler. + inset = None + if re.search(r'UIEdgeInsetsMake', line) is not None: + result = inset_re.search(re.sub(r'\s','',line)) + assert result is not None, 'Found UIEdgeInsetsMake, but could not parse arguments: %r' % line + assert len(result.groups()) == 4, 'invalid inset call: %r' % line + inset = tuple([float(x) for x in result.groups()]) + + return (name, inset) + + filenames = dict() + for f in source_files: + contents = open(f, 'r').read() + for i in image_re.findall(contents): + name, inset = ParseLazyStaticImage(i) + + if name in filenames: + assert filenames[name] == inset, 'Image found with different insets: %s' % name + else: + filenames[name] = inset + + # Adjust for images that aren't explicitly referenced in the code. + for name in FORCE_REFERENCE: + filenames[name] = None + + return filenames + + +def FindMissingImages(referenced_images, asset_images): + """ Check that every referenced image (and its 2x version) is found in 'asset_images'. """ + images = set(asset_images) + for ref in referenced_images: + if ref not in images: + print '%s does not exist' % ref + if ImageName2X(ref) not in images: + print '%s does not exist' % ref_2x + + +def GetInsetFrom9Patch(path): + """ Given the path to a 9-patch image, extract the regions and return an iOS inset specification: + (top, left, bottom, right) in points. + Android 9-patch is a regular image with an extra 1 pixel frame. Each frame pixel has value either 0 or 255. + See: http://developer.android.com/guide/topics/graphics/2d-graphics.html#nine-patch + The black lines in the example are actually present in the file, they are stored in the 1 pixel frame. + + We extract those lines and count the number of pixels before and after the area, which gives us the iOS + specification passed to UIEdgeInsetsMake. + """ + img = Image.open(path) + pixels = img.load() + width, height = img.size + vert_line = [ pixels[0, y][3] for y in xrange(height) ] + hor_line = [ pixels[x, 0][3] for x in xrange(width) ] + # We want the real image limits, so remove the first and last row/col. + vert_line = vert_line[1:-1] + hor_line = hor_line[1:-1] + + # Find the size of each group of "0" or "255" in the extra pixels. + vert_groups = [(k, len(list(g))) for k, g in itertools.groupby(vert_line)] + hor_groups = [(k, len(list(g))) for k, g in itertools.groupby(hor_line)] + + assert len(vert_groups) <= 3, vert_groups + assert len(hor_groups) <= 3, hor_groups + + # Generate iOS-style boundaries: (delta top, delta left, delta bottom, delta right) + # Since we're operating on 2x images, we devide by two to get values in points instead of pixels. + top = left = bottom = right = 0 + if vert_groups[0][0] == 0: + top = float(vert_groups[0][1]) / 2 + if vert_groups[-1][0] == 0: + bottom = float(vert_groups[-1][1]) / 2 + if hor_groups[0][0] == 0: + left = float(hor_groups[0][1]) / 2 + if hor_groups[-1][0] == 0: + right = float(hor_groups[-1][1]) / 2 + return (top, left, bottom, right) + + +def Generate9PatchFromInset(src_path, src_inset, dest_path, just_print_errors): + """ Generate an android 9-patch at 'dest_path' from a source image at 'src_path' using the iOS inset 'src_inset'. """ + src_img = Image.open(src_path) + width, height = src_img.size + + # Create a new image with the same mode but 1-pixel frame. + new_width = width + 2 + new_height = height + 2 + new_img = Image.new('RGBA', size=(new_width, new_height), color=(0, 0, 0, 0)) + + src_pixels = src_img.load() + new_pixels = new_img.load() + # Copy the source pixels into the dest image with an offset of (1, 1). + # The paste command does weird things with the alpha band, so we do it manually. + is_rgb = src_img.mode == 'RGB' + for x in xrange(width): + for y in xrange(height): + pixel = src_pixels[x, y] + if is_rgb: + # If the source image does not have an alpha channel, we need to add one. + assert len(pixel) == 3, 'RGB image with %d channels' % len(pixel) + new_pixels[x + 1, y + 1] = pixel + (255,) + else: + assert len(pixel) == 4, 'RGBA image with %d channels' % len(pixel) + new_pixels[x + 1, y + 1] = pixel + + # Go through the src inset. Multiple by two to convert from points to pixels in the 2x image. + top, left, bottom, right = [int(x * 2) for x in src_inset] + # Look for the size of the region. width - left - right and height - top - bottom + region_hor = width - left - right + region_ver = height - top - bottom + print ' width=%f, left=%f, right=%f' % (width, left, right) + print ' height=%f, top=%f, bottom=%f' % (height, top, bottom) + if (region_hor <= 0 or region_ver <= 0): + print ' ERROR: scaling region has width or height <= 0' + return + + assert region_hor > 0 + assert region_ver > 0 + + # We set the scale pixels as: 0, 0 * left, 255 * region_hor, 0 * right, 0 + # We set the fill pixels as 0, 0 * width, 0 + # The start and end 0 are for the 1-pixel frame. + line_hor_scale = [0] + ([0] * left) + ([255] * region_hor) + ([0] * right) + [0] + line_hor_fill = [0] + ([255] * width) + [0] + line_ver_scale = [0] + ([0] * top) + ([255] * region_ver) + ([0] * bottom) + [0] + line_ver_fill = [0] + ([255] * height) + [0] + + assert len(line_hor_scale) == new_width + assert len(line_hor_fill) == new_width + assert len(line_ver_scale) == new_height + assert len(line_ver_fill) == new_height + + # Set the pixels in the frame. We specify the android 9-patch "fill" region to be the entire image. + for y in xrange(new_height): + new_pixels[0, y] = (0, 0, 0, line_ver_scale[y]) + new_pixels[new_width - 1, y] = (0, 0, 0, line_ver_fill[y]) + for x in xrange(new_width): + new_pixels[x, 0] = (0, 0, 0, line_hor_scale[x]) + new_pixels[x, new_height - 1] = (0, 0, 0, line_hor_fill[x]) + + if not just_print_errors: + print ' Writing 9-patch: %s with size: %r' % (dest_path, new_img.size) + new_img.save(dest_path) + + +def GenerateAndroid9Patch(referenced_images): + """ Iterate over all referenced images with insets and check against the android 9-patch images. """ + for name, inset in referenced_images.iteritems(): + if inset is None: + continue + + img_path = os.path.join(IOS_IMAGES_DIR, ImageName2X(name)) + assert os.access(img_path, os.R_OK), '2x version of %s is not readable' % name + + nine_path = os.path.join(ANDROID_IMAGES_DIR, ImageName9Patch(name)) + generate_9patch = True + if os.access(nine_path, os.R_OK): + generate_9patch = False + android_inset = GetInsetFrom9Patch(nine_path) + if inset == android_inset: + if options.options.v: + print '%s: OK %r' % (name, inset) + else: + print '%s: 9-patch with different inset:' % name + print ' iOS 2x: size: %r, inset: %r' % (Image.open(img_path).size, inset) + print ' android: size: %r, inset: %r' % (Image.open(nine_path).size, android_inset) + # Dry run to print any errors. + Generate9PatchFromInset(img_path, inset, nine_path, True) + if name in options.options.regenerate: + print ' Regenerating 9 patch...' + generate_9patch = True + if generate_9patch: + print '%s: generating 9-patch with inset %r' % (name, inset) + print ' iOS 2x: size: %r, inset: %r' % (Image.open(img_path).size, inset) + Generate9PatchFromInset(img_path, inset, nine_path, False) + + +def FindMissingAndroidPlainAssets(referenced_images, android_assets): + """ Iterate over all referenced images in iOS. For each plain asset (without an inset), check whether it + exists in the android assets list. + """ + for name, inset in referenced_images.iteritems(): + if inset is not None: + continue + if name in SKIP_PLAIN: + print 'WARNING: %s: skipping due to hard-coded exclusion in assets-tool.py' % name + continue + android_name = ImageNameAndroid(name) + if android_name not in android_assets: + name_2x = ImageName2X(name) + shutil.copyfile(os.path.join(IOS_IMAGES_DIR, name_2x), os.path.join(ANDROID_IMAGES_DIR, android_name)) + print '%s: not in android assets, copied %s -> %s' % (name, name_2x, android_name) + else: + if options.options.v: + print '%s: OK' % name + +if __name__ == '__main__': + options.parse_command_line(final=True) + # Images found in the code. dict of "filename" -> "inset quad" + referenced_images = FindReferencedIOSImages() + # Full list of files in the images directory, filenames only. + asset_images = ListBasename(FindFilesIn(IOS_IMAGES_DIR, '*')) + + FindMissingImages(sorted(referenced_images.keys()), asset_images) + + if options.options.ninepatch: + GenerateAndroid9Patch(referenced_images) + if options.options.plain: + android_assets = ListBasename(FindFilesIn(ANDROID_IMAGES_DIR, '*')) + FindMissingAndroidPlainAssets(referenced_images, android_assets) diff --git a/clients/shared/scripts/developer-defines-gyp.sh b/clients/shared/scripts/developer-defines-gyp.sh new file mode 100644 index 0000000..6f7c4df --- /dev/null +++ b/clients/shared/scripts/developer-defines-gyp.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +function init_defines() { + local out="${1}" + local base="$(basename ${out})" + if test "${base}" = ".viewfinder.DeveloperDefines.h"; then + cat > "${out}" < "${out}" </dev/null 2>&1; then + mkdir -p $(dirname "${out}") + ln -f "${in}" "${out}" + fi +} + +# Have to be careful about quoting these variables - gyp likes to use spaces +# in intermediate directory names. +for i in DeveloperDefines.h TestDefines.h; do + link_defines "$i" +done diff --git a/clients/shared/scripts/protoc-gyp.sh b/clients/shared/scripts/protoc-gyp.sh new file mode 100755 index 0000000..a530f03 --- /dev/null +++ b/clients/shared/scripts/protoc-gyp.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +OS_NAME=$(uname -s) +PROTOC=$(dirname $0)/../../../third_party/shared/bin/protoc.${OS_NAME} + +# If the path to the .proto file is relative, protoc includes the directory +# component of the path in the output directory. We squash this behavior be +# prepending PWD. +proto_path="${PWD}/$2" + +# protoc requires that we always include a -I for the directory containing +# the .proto being processed, hence the $(dirname ${proto_path}). +declare -a include_dirs +include_dirs=( $1 $(dirname ${proto_path}) ) +proto_includes=${include_dirs[@]/#/-I} + +out_dir="$3" + +# echo ${PROTOC} --cpp_out="${out_dir}" ${proto_includes} "${proto_path}" >> ${HOME}/protoc.out +${PROTOC} --cpp_out="${out_dir}" ${proto_includes} "${proto_path}" diff --git a/clients/shared/shared.android.gyp b/clients/shared/shared.android.gyp new file mode 100644 index 0000000..f59562a --- /dev/null +++ b/clients/shared/shared.android.gyp @@ -0,0 +1,124 @@ +{ + 'targets': [ + { + 'target_name': 'sharedprotos', + 'type': 'static_library', + 'includes': [ + 'protoc.gypi', + ], + 'dependencies': [ + '../../third_party/shared/protobuf.gyp:libprotobuf', + ], + 'sources': [ + ' + + viewfinder + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + + + README + 1 + VF_LOC/README + + + TEAM + 1 + VF_LOC/TEAM + + + __init__.py + 1 + VF_LOC/__init__.py + + + backend + 2 + VF_LOC/backend + + + keyczar + 2 + /Users/andy/viewfinder/third_party/Darwin-11.0.0-x86_64-i386-64bit/lib/python2.7/site-packages/keyczar + + + scripts + 2 + VF_LOC/scripts + + + secrets + 2 + VF_LOC/secrets + + + tornado + 2 + $%7BVF_LOC%7D/third_party/Darwin-11.0.0-x86_64-i386-64bit/lib/python2.7/site-packages/tornado-2.2-py2.7.egg/tornado + + + + + 1366262739453 + + 30 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-*.pyc + + + + 1339132309530 + backend + 26 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-experimental + + + + 1339132309531 + backend + 26 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-cockroach + + + + + + VF_LOC + $%7BPARENT-2-PROJECT_LOC%7D + + + diff --git a/eclipse/backend-project/.pydevproject b/eclipse/backend-project/.pydevproject new file mode 100644 index 0000000..593b7b5 --- /dev/null +++ b/eclipse/backend-project/.pydevproject @@ -0,0 +1,11 @@ + + + + +python 2.7 +Default + + +${VF_LOC}/.. + + diff --git a/experimental/fogbugz_export/fogbugz_export.py b/experimental/fogbugz_export/fogbugz_export.py new file mode 100644 index 0000000..0e9422d --- /dev/null +++ b/experimental/fogbugz_export/fogbugz_export.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +"""Exports fogbugz cases to a file. + +Create a text file ~/.fogbugz containing your credentials in json: + {"email": "ben@emailscrubbed.com", "password": "asdf"} + +The fogbugz API is insane: the API doesnt let you specify queries directly; it uses +a single persistent per-user query (same as the website). Before running this script +go to the fogbugz website and set your current view to the query you want to run +(e.g. "all open cases"). +""" + +import json +import os +import urllib +from xml.etree import ElementTree + +from tornado import gen +from tornado.httpclient import AsyncHTTPClient +from tornado.ioloop import IOLoop +from tornado.options import define, options, parse_command_line + +define('base_url', default='https://viewfinder.fogbugz.com/api.asp') +define('credential_file', default=os.path.expanduser('~/.fogbugz')) +define('output_file', default='cases.xml') + +def make_url(**kwargs): + return options.base_url + '?' + urllib.urlencode(kwargs) + +@gen.coroutine +def fetch(**kwargs): + http = AsyncHTTPClient() + response = yield http.fetch(make_url(**kwargs)) + raise gen.Return(ElementTree.fromstring(response.body)) + +@gen.coroutine +def main(): + parse_command_line() + + with open(options.credential_file) as f: + credentials = json.load(f) + + response = yield fetch(cmd='logon', email=credentials['email'], password=credentials['password']) + + token = response.find('token').text + + cases = yield fetch(cmd='search', token=token, + cols='sTitle,sPersonAssignedTo,sProject,sArea,sCategory,ixPriority,sPriority,events') + + with open(options.output_file, 'w') as f: + f.write(ElementTree.tostring(cases)) + +if __name__ == '__main__': + IOLoop.instance().run_sync(main) diff --git a/experimental/fogbugz_export/parse_cases.py b/experimental/fogbugz_export/parse_cases.py new file mode 100644 index 0000000..da7f4b8 --- /dev/null +++ b/experimental/fogbugz_export/parse_cases.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +"""Converts fogbugz xml into a more useful format. + +When run from the command line, prints the data. May also be imported for the parse_cases function. +""" + +from tornado import escape +from tornado.ioloop import IOLoop +from tornado.options import define, options, parse_command_line +from xml.etree import ElementTree + +# Maps fogbugz users to bitbucket users. +USER_MAP = { + 'CLOSED': '', + 'All Engineers': '', + 'Andrew Kimball': 'andy_kimball', + 'Ben Darnell': 'bdarnell', + 'Brett Eisenman': 'beisenman', + 'Brian McGinnis': 'jbrianmcg', + 'Chris Schoenbohm': 'cschoenbohm', + 'Dan Shin': 'danshin', + 'Harry Clarke': 'hsclarke', + 'Marc Berhault': 'mberhault', + 'Matt Tracy': 'bdarnell', # assign matt's bugs to ben until he gets back from vacation and sets up his account + 'Mike Purtell': 'mikepurt', + 'Peter Mattis': 'peter_mattis', + 'Spencer Kimball': 'spencerkimball', +} + +def parse_cases(filename): + """Parses the fogbugz data in the file. + + Returns a list of (subject, assigned_to, body) tuples. + """ + results = [] + + tree = ElementTree.parse(filename) + + for case in tree.find('cases').findall('case'): + subject = 'FB%s: %s' % (case.get('ixBug'), case.findtext('sTitle')) + body = [] + assigned_to = case.findtext('sPersonAssignedTo') + body.append('Assigned to: %s' % assigned_to) + body.append('Project: %s' % case.findtext('sProject')) + body.append('Area: %s' % case.findtext('sArea')) + body.append('Priority: %s (%s)' % (case.findtext('ixPriority'), case.findtext('sPriority'))) + body.append('Category: %s' % case.findtext('sCategory')) + body.append('') + for event in case.find('events').findall('event'): + body.append( '%s at %s' % (event.findtext('evtDescription'), event.findtext('dt'))) + if event.findtext('s'): + body.append('') + body.append(event.findtext('s')) + body.append('') + if event.find('rgAttachments') is not None: + for attachment in event.find('rgAttachments').findall('attachment'): + body.append('Attachment: %s' % escape.xhtml_unescape(attachment.findtext('sURL'))) + results.append((subject, USER_MAP[assigned_to], '\n'.join(body))) + return results + +def main(): + define('filename', default='cases.xml') + parse_command_line() + + for subject, assigned_to, body in parse_cases(options.filename): + print subject + print 'assigned to: ', assigned_to + print + print body + print + +if __name__ == '__main__': + IOLoop.instance().run_sync(main) diff --git a/experimental/fogbugz_export/upload_issues.py b/experimental/fogbugz_export/upload_issues.py new file mode 100644 index 0000000..d22df90 --- /dev/null +++ b/experimental/fogbugz_export/upload_issues.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +"""Uploads issues to bitbucket. +""" + +import urllib + +from tornado.escape import utf8 +from tornado import gen +from tornado.httpclient import AsyncHTTPClient +from tornado.ioloop import IOLoop +from tornado.options import define, options, parse_command_line + +import parse_cases + +define('filename', default='cases.xml') +define('repo', default='viewfinder/viewfinder') +define('username', default='viewfinder') +define('password', default='') + +@gen.coroutine +def main(): + parse_command_line() + assert options.password + client = AsyncHTTPClient() + + cases = parse_cases.parse_cases(options.filename) + + url = 'https://api.bitbucket.org/1.0/repositories/%s/issues' % options.repo + for subject, assigned_to, body in cases: + args = {'title': utf8(subject), 'content': utf8(body)} + if assigned_to: + args['responsible'] = assigned_to + response = yield client.fetch(url, method='POST', body=urllib.urlencode(args), + auth_username=options.username, auth_password=options.password) + print response, response.body + +if __name__ == '__main__': + IOLoop.instance().run_sync(main) diff --git a/experimental/imagefingerprint/ImageFingerprint.cc b/experimental/imagefingerprint/ImageFingerprint.cc new file mode 120000 index 0000000..624463e --- /dev/null +++ b/experimental/imagefingerprint/ImageFingerprint.cc @@ -0,0 +1 @@ +../../clients/ios/Source/ImageFingerprint.cc \ No newline at end of file diff --git a/experimental/imagefingerprint/ImageFingerprint.h b/experimental/imagefingerprint/ImageFingerprint.h new file mode 120000 index 0000000..439e556 --- /dev/null +++ b/experimental/imagefingerprint/ImageFingerprint.h @@ -0,0 +1 @@ +../../clients/ios/Source/ImageFingerprint.h \ No newline at end of file diff --git a/experimental/imagefingerprint/imagefingerprintmodule.cc b/experimental/imagefingerprint/imagefingerprintmodule.cc new file mode 100644 index 0000000..7363dfd --- /dev/null +++ b/experimental/imagefingerprint/imagefingerprintmodule.cc @@ -0,0 +1,57 @@ +#include +#include +#include +#include +#include "ImageFingerprint.h" + +using std::string; +using std::vector; + +static PyObject* PyFingerprintImage(PyObject* self, PyObject* args) { + char* filename; + if (!PyArg_ParseTuple(args, "s", &filename)) { + return NULL; + } + + CGDataProviderRef provider = CGDataProviderCreateWithFilename(filename); + if (!provider) { + PyErr_SetString(PyExc_Exception, "error creating CGDataProvider"); + return NULL; + } + CGImageSourceRef source = CGImageSourceCreateWithDataProvider(provider, NULL); + if (!source) { + CFRelease(provider); + PyErr_SetString(PyExc_Exception, "error creating CGImageSource"); + return NULL; + } + CGImageRef image = CGImageSourceCreateImageAtIndex(source, 0, NULL); + if (!image) { + CFRelease(source); + CFRelease(provider); + PyErr_SetString(PyExc_Exception, "error creating CGImage"); + return NULL; + } + + vector fingerprint = FingerprintImage(image); + + CFRelease(image); + CFRelease(source); + CFRelease(provider); + + PyObject* list = PyList_New(fingerprint.size()); + for (int i = 0; i < fingerprint.size(); i++) { + PyList_SET_ITEM(list, i, PyString_FromStringAndSize(fingerprint[i].data(), + fingerprint[i].size())); + } + + return list; +}; + +static PyMethodDef kMethods[] = { + {"FingerprintImage", PyFingerprintImage, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL}, +}; + +PyMODINIT_FUNC initimagefingerprint(void) { + Py_InitModule("imagefingerprint", kMethods); +} diff --git a/experimental/imagefingerprint/setup.py b/experimental/imagefingerprint/setup.py new file mode 100644 index 0000000..1578183 --- /dev/null +++ b/experimental/imagefingerprint/setup.py @@ -0,0 +1,17 @@ +from distutils.core import setup, Extension + +extension = Extension( + name='imagefingerprint', + sources=['ImageFingerprint.cc', + 'imagefingerprintmodule.cc', + ], + extra_compile_args=['-Wno-unused-variable'], + extra_link_args=['-framework', 'CoreGraphics', + '-framework', 'Accelerate', + '-framework', 'ImageIO', + ], + ) + +setup(name='imagefingerprint', + version='0.1', + ext_modules=[extension]) diff --git a/experimental/imagefingerprint/tools/find_near_dupes.py b/experimental/imagefingerprint/tools/find_near_dupes.py new file mode 100755 index 0000000..36a9615 --- /dev/null +++ b/experimental/imagefingerprint/tools/find_near_dupes.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +"""Reads a fingerprint file and prints all near-duplicate images. + +This is a python reimplementation of the indexing in ImageIndex.mm. + +The input is a tab-separated file as produced by fingerprint_directory.py. +The output is: filename1 TAB filename2 TAB hamming distance. +""" + +import collections +from tornado.options import parse_command_line, options, define + +define('min', default=0) +define('max', default=12) + +def parse_fp(fp): + return [s.decode('hex') for s in fp.strip().split(':')] + +kTagLengths = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4] + +def gen_tags(fp): + for term in fp: + term = term.encode('hex') + pos = 0 + for l in kTagLengths: + yield '%d:%s' % (pos, term[pos:pos+l]) + pos += l + assert pos == len(term) + +def count_bits(n): + return bin(n).count('1') + +def hamming(t1, t2): + assert len(t1) == len(t2) + total = 0 + for c1, c2 in zip(t1, t2): + d = ord(c1) ^ ord(c2) + total += count_bits(d) + return total + +def distance(fp1, fp2): + return min(hamming(t1, t2) for t1 in fp1 for t2 in fp2) + +class Index(object): + def __init__(self): + # maps tags -> list of filename, fingerprint pairs + self.index = collections.defaultdict(list) + + def load(self, filename): + with open(filename) as f: + for line in f: + filename, fingerprint = line.split('\t') + fingerprint = parse_fp(fingerprint) + for tag in gen_tags(fingerprint): + self.index[tag].append((filename, fingerprint)) + + def find_matches(self, min_hamming, max_hamming): + assert min_hamming <= max_hamming + assert max_hamming <= 12, 'accuracy not guaranteed for distance > 12' + seen = set() + for tag, lst in self.index.iteritems(): + for fn1, fp1 in lst: + for fn2, fp2 in lst: + if fn1 == fn2: + continue + key = tuple(sorted([fn1, fn2])) + if key in seen: + continue + seen.add(key) + dist = distance(fp1, fp2) + if min_hamming <= dist <= max_hamming: + print '%s\t%s\t%d' % (fn1, fn2, dist) + +def main(): + args = parse_command_line() + + index = Index() + for arg in args: + index.load(arg) + + index.find_matches(options.min, options.max) + +if __name__ == '__main__': + main() diff --git a/experimental/imagefingerprint/tools/fingerprint_directory.py b/experimental/imagefingerprint/tools/fingerprint_directory.py new file mode 100755 index 0000000..7c23e78 --- /dev/null +++ b/experimental/imagefingerprint/tools/fingerprint_directory.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +"""Walks a directory, fingerprinting all .jpg files therein. + +Output is a tab-separated file: filename TAB fingerprint +""" + +import os + +from tornado.options import parse_command_line + +from imagefingerprint import FingerprintImage + +def format_fp(fp): + return ':'.join(s.encode('hex') for s in fp) + +def main(): + args = parse_command_line() + + for arg in args: + for dirpath, dirnames, filenames in os.walk(arg): + for filename in filenames: + if filename.lower().endswith('.jpg'): + qualname = os.path.join(dirpath, filename) + assert '\t' not in qualname + fp = FingerprintImage(qualname) + print '%s\t%s' % (qualname, format_fp(fp)) + +if __name__ == '__main__': + main() diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 0000000..a7f8998 --- /dev/null +++ b/fabfile.py @@ -0,0 +1,767 @@ +"""Viewfinder production scripts + +Cookbook: + +* Launch new instances: specify NodeType,Name,Zone + $ fab create_instance:STAGING,STAGING_003,us-east-1c +* Resume installation of failed instance creation (eg: stuck in state 'pending' or not ssh-able): + $ fab create_instance:STAGING,STAGING_003,us-east-1c,i-9a0cdbe9 +* Stop and destroy a running instance: + $ fab destroy_instance:STAGING,i-9a0cdbe9 +* Deploy env/code changes and restart: + $ fab nodetype:STAGING deploy + $ fab nodetype:PROD deploy + +Region currently default to 'us-east-1'. When running in multiple regions, add region:<...> task. + +""" +import os +import re +import subprocess +import time +from collections import defaultdict +from fabric.api import * +from fabric.operations import * +from fabric.network import NetworkError +from fabric.state import output +from fabric.utils import * +from fabric.contrib.files import * +from fabric.contrib.project import rsync_project + +from viewfinder.backend.base import util +from viewfinder.backend.prod import ec2_utils + +kVFPassphraseFile = '~/.ssh/vf-passphrase' +kInstanceType = 'm1.medium' + +env.user = 'ec2-user' +env.key_filename = '~/.ssh/wwwkey.pem' +env.region = 'us-east-1' +env.node_type = None + +# Disable various output levels. +output['running'] = False +output['stdout'] = False +output['warnings'] = False + +# Amazon Linux AMI. Note that these are region-specific; this one is for +# us-east. List is at http://aws.amazon.com/amazon-linux-ami/ +BASE_AMI = 'ami-3275ee5b' + +def runs_last(func): + """Decorator to run a function only on the last invocation. + We determine last by comparing the number times called with the size of env.hosts. + Return None on all invocations but the last one where we return the function return value. + """ + def Wrapper(): + calls = func.num_host_calls + if calls >= len(env.hosts) - 1: + return func() + else: + func.num_host_calls = calls + 1 + return None + + setattr(func, 'num_host_calls', 0) + return Wrapper + + +def fprint(string): + host_str = '[%s] ' % env.host_string if env.host_string else '' + time_str = time.strftime("%H:%M:%S") + puts('%s%s %s' % (host_str, time_str, string), show_prefix=False, end='\n') + + +def fprompt(text, default=None, validate=None): + host_str = '[%s] ' % env.host_string if env.host_string else '' + time_str = time.strftime("%H:%M:%S") + return prompt('%s%s %s' % (host_str, time_str, text), default=default, validate=validate) + + +def load_passphrase_from_file(): + """Read the viewfinder passphrase from local file.""" + vf_path = os.path.expanduser(kVFPassphraseFile) + assert os.access(vf_path, os.F_OK) and os.access(vf_path, os.R_OK), '%s must exist and be readable' % vf_path + with open(vf_path) as f: + user_data = f.read() + return user_data.strip('\n') + + +def get_ami_metadata(): + """Fetch ami metadata for the local instance. Returns a dict of 'key':'value'. eg: 'instance-id':'i-e7f7e69b'.""" + res = {} + base_url = 'http://169.254.169.254/latest/meta-data' + instance_id = run('curl %s/instance-id' % base_url) + assert re.match('i-[0-9a-f]{8}', instance_id) + res['instance-id'] = instance_id + + return res + + +@task +def get_healthz(): + """Fetch healthz status for the local instance.""" + url = 'https://localhost:8443/healthz' + ret = 'FAIL' + with settings(warn_only=True): + ret = run('curl -k %s' % url) + fprint('Healthz status: %s' % ret) + return ret == 'OK' + +@task +def nodetype(typ): + """Specify node type: STAGING or PROD.""" + # Don't override hosts if specified on the command line. + if not env.hosts: + env.hosts = ec2_utils.ListInstancesDNS(region='us-east-1', node_types=[typ], states=['running']) + env.nodetype = typ + + +def is_old_env(): + """Return True if ~/env is old-style (plain directory) or False if new style (symlink). + No ~/env returns False. + """ + env_exists = exists('~/env') + if not env_exists: + # So such directory or link. + return False + with settings(warn_only=True): + is_link = run('readlink ~/env') + if is_link.return_code == 0: + # This is a symlink. New-style environment. + return False + return True + + +def is_old_code(): + """Return True if ~/viewfinder is old-style (plain directory) or False if new style (symlink). + No ~/viewfinder returns False. + """ + code_exists = exists('~/viewfinder') + if not code_exists: + # So such directory or link. + return False + with settings(warn_only=True): + is_link = run('readlink ~/viewfinder') + if is_link.return_code == 0: + # This is a symlink. New-style code. + return False + return True + + +def get_file_suffix(prefix, filename): + # Depending on how the linking was done, the destination could be absolute or relative, with or without '/'. + result = re.match(r'^(?:/home/%s/)?%s.([0-9a-f]+)/?$' % (env.user, prefix), filename) + if result is None or len(result.groups()) != 1: + return None + return result.groups()[0] + + +def get_link_suffix(symlink): + """Follow 'symlink' in ~/ and determine the suffix for the target (of the form .[a-f0-9]+. + Returns None if the symlink does not exist or is not a symlink. + """ + with settings(warn_only=True): + if not exists('~/%s' % symlink): + return None + target = run('readlink ~/%s' % symlink) + if target.return_code != 0: + return None + suffix = get_file_suffix(symlink, target) + assert suffix is not None, 'Could not determine suffix from filename %s' % target + return suffix + + +def active_env(): + """Return the revision ID of the current environment, or None if it does not exist or cannot be determined.""" + return get_link_suffix('env') + + +def active_code(): + """Return the revision ID of the current code, or None if it does not exist or cannot be determined.""" + return get_link_suffix('viewfinder') + + +def latest_requirements_revision(): + """Return the revision ID of the last change to the prod-requirements file. + hg log lists all revisions, regardless of what we're synced to. -r :. shows all entries up to the currently-synced + point. However, they are listed in reverse order (older first, latest last), so we must tail it. + """ + return local('hg log -r :. --template "{node|short}\n" scripts/prod-requirements.txt | tail -n 1', capture=True) + + +def hg_revision(): + """Returns the HG revision.""" + return local('hg identify -i', capture=True) + + +def hg_revision_timestamp(rev): + """Returns the timestamp (in seconds) of 'rev', or None if cannot be determined. + + Since mq (and non-linear history in general) makes it possible have revisions dated + before their true "commit" date, we must find the newest ancestor of the given revision. + """ + try: + revset = 'last(sort(ancestors(%s), date))' % rev + res = subprocess.check_output(['hg', 'log', '-r', revset, '--template', '{date}'], stderr=subprocess.STDOUT) + return float(res.strip()) + except subprocess.CalledProcessError: + return None + + +@runs_once +def code_prep(): + """Generate the code tarball and return the HG revision.""" + rev = hg_revision() + assert not rev.endswith('+'), 'Client has pending changes, cannot install.' + fprint('Preparing local code tarball (rev %s)' % rev) + filename = 'viewfinder.%s.tar.gz' % rev + + local('hg identify -i > hg_revision.txt') + local('tar czf %s --exclude "*.o" --exclude "*~" --exclude "*.pyc" __init__.py scripts/ marketing/ backend/ resources/ secrets/viewfinder.co hg_revision.txt' % filename) + return rev + + +@runs_last +def code_cleanup(): + """Delete the generated tarball and revision file.""" + fprint('Cleaning up local code') + local('rm -f hg_revision.txt viewfinder.*.tar.gz') + + +@task +def code_install(): + """Install latest code from local directory. + We put the current hg revision in a file, generate a local tarball, copy it to the instance and untar. + code_prep() and code_cleanup() are run the first and last time respectively. + """ + assert env.host_string, "no hosts specified" + assert not is_old_code(), 'Active code is using the old style (directory instead of symlink). ' \ + 'Manual intervention required' + # code_prep is only run the first time. Subsequent runs return the same value as the first time. + rev = code_prep() + if code_verify(rev): + return + + fprint('Installing code (rev %s)' % rev) + + filename = 'viewfinder.%s.tar.gz' % rev + dirname = 'viewfinder.%s' % rev + + put(filename, '~/%s' % filename) + + run('mkdir -p ~/%s' % dirname) + # TODO: purge old pycs + with cd('~/%s' % dirname): + run('tar xzvf ../%s' % filename) + # HACK: the local viewfinder/pythonpath directory has testing garbage in it, + # so until we fix the push to use the hg manifest recreate it on the other + # side instead of syncing it. + run('mkdir -p ~/%s/pythonpath' % dirname) + with cd('~/%s/pythonpath' % dirname): + run('ln -f -s ~/%s viewfinder' % dirname) + + # Delete the tarball. We never reuse it anyway. + run('rm -f ~/%s' % filename) + + # code_cleanup is run on the last invocation (based on the size of env.hosts). + code_cleanup() + + +@task +def code_activate(requirements_revision=None): + """Make the code at revision active (latest if None).""" + assert not is_old_code(), 'Active code is old-style (directory, not symlink). Manual intervention required!' + req_rev = requirements_revision or hg_revision() + assert code_verify(req_rev), 'Desired code revision %s invalid, cannot be made active' % req_rev + # Note: -T forces the target to be treated as a normal file. Without it, the link will be: + # ~/viewfinder/viewfinder. -> ~/viewfinder. instead of being in the home directory. + run('ln -T -s -f ~/viewfinder.%s ~/viewfinder' % req_rev) + fprint('Code at revision %s marked active.' % req_rev) + + +@task +def code_verify(revision=None): + """Verify the code for a given revision (latest if None). + We only check the symlink. TODO: find a way to validate the code itself.""" + if is_old_code(): + fprint('installed code is in the old style (directory instead of symlink). Manual intervention required') + return False + rev = revision or hg_revision() + if exists('~/viewfinder.%s' % rev): + fprint('Code at revision %s is installed' % rev) + return True + else: + fprint('Code at revision %s is not installed' % rev) + return False + +@task +def virtualenv_install(): + """Install the latest virtual environment if needed. + We do nothing if the env is already the latest. + We do install the new environment even if we are using the old style. + This does not activate (symlink) the newly installed environment. + """ + # Installs the latest virtual environment from the local prod-requirements.txt. + prod_rev = latest_requirements_revision() + assert re.match(r'[0-9a-f]+', prod_rev) + + active_env_rev = active_env() + if prod_rev == active_env_rev: + assert virtualenv_verify(prod_rev), 'Active environment is not valid' + return + + env_dir = 'env.%s' % prod_rev + package_dir = 'python-package.%s' % prod_rev + requirements_file = 'prod-requirements.txt.%s' % prod_rev + if exists(env_dir): + fprint('prod-requirements (rev %s) already installed, but not active.' % prod_rev) + else: + fprint('installing environment from prod-requirements (rev %s)' % prod_rev) + run('rm -rf ~/%s ~/%s ~/%s' % (env_dir, package_dir, requirements_file)) + rsync_project(local_dir='third_party/python-package/', remote_dir='~/%s/' % package_dir, ssh_opts='-o StrictHostKeyChecking=no') + put('scripts/prod-requirements.txt', '~/%s' % requirements_file) + run('python2.7 ~/%s/virtualenv.py --never-download ~/%s/viewfinder' % (package_dir, env_dir)) + + # Let fabric surface the failure. + run('~/%s/viewfinder/bin/pip install -f file://$HOME/%s --no-index -r ~/%s' % + (env_dir, package_dir, requirements_file)) + # Do not delete the prod-requirements file when done as we may use it to verify the environment later. + + +@task +def virtualenv_activate(requirements_revision=None): + """Make the virtual env at revision active (latest if None).""" + assert not is_old_env(), 'Active environment is old-style (directory, not symlink). Manual intervention required!' + req_rev = requirements_revision or latest_requirements_revision() + assert virtualenv_verify(req_rev), 'Desired env revision %s invalid, cannot be made active' % req_rev + + # Create sitecustomize.py file, which sets default str encoding as UTF-8. + # See http://blog.ianbicking.org/illusive-setdefaultencoding.html. + env_dir = 'env.%s' % req_rev + run('echo "import sys; sys.setdefaultencoding(\'utf-8\')" > %s/viewfinder/lib/python2.7/sitecustomize.py' % env_dir); + + # Note: -T forces the target to be treated as a normal file. Without it, the link will be: + # ~/viewfinder/viewfinder. -> ~/viewfinder. instead of being in the home directory. + run('ln -T -s -f ~/env.%s ~/env' % req_rev) + fprint('Environment at rev %s marked active.' % req_rev) + + +@task +def virtualenv_verify(requirements_revision=None): + """Verify the virtual environment for a given revision (latest if None).""" + req_rev = requirements_revision or latest_requirements_revision() + + env_dir = 'env.%s' % req_rev + package_dir = 'python-package.%s' % req_rev + requirements_file = 'prod-requirements.txt.%s' % req_rev + with settings(warn_only=True): + out = run('~/%s/viewfinder/bin/pip install -f file://$HOME/%s --no-index -r ~/%s --no-install --no-download -q' % (env_dir, package_dir, requirements_file)) + if out.return_code == 0: + fprint('Valid virtual environment for prod-requirements (rev %s)' % req_rev) + return True + else: + fprint('Bad virtual environment for prod-requirements (rev %s)' % req_rev) + return False + + +@task +def install_crontab(): + """Install or remove crontab for given node type.""" + assert env.nodetype, 'no nodetype specified' + assert env.host_string, 'no hosts specified' + cron_file = '~/viewfinder/scripts/crontab.%s' % env.nodetype.lower() + # Run 'crontab ' if the remote file exists, otherwise run 'crontab -r'. + # Warn only as 'crontab -r' fails if no crontab is installed. + with settings(warn_only=True): + run('if [ -e %s ]; then crontab %s; else crontab -r; fi' % (cron_file, cron_file)) + + +@task +def yum_install(): + """Install required yum packages.""" + fprint('Installing yum packages.') + sudo('yum -y update') + sudo('yum -y install make zlib gcc gcc-c++ openssl-devel python27 python27-devel libcurl-devel pcre-devel') + + +@task +def haproxy_install(): + """Install and configure haproxy. + HAProxy is not controlled by the prod-requirements file, and not easily versioned. As such, we install it in its + own directory. + TODO(marc): replace with yum package once 1.5 is stable and rolled out to AWS. + """ + # rsync the haproxy source. + fprint('Rsync thirdparty/haproxy ~/haproxy') + rsync_project(local_dir='third_party/haproxy/', remote_dir='~/haproxy/', ssh_opts='-o StrictHostKeyChecking=no') + + # build haproxy and install it in ~/bin.} + fprint('Building haproxy') + run('haproxy/build.sh ~/') + + # Concatenate the certificate and key into a single file (this is expected by haproxy) and push it. + fprint('Generating viewfinder.pem for haproxy') + vf_passphrase = load_passphrase_from_file() + # Staging and prod use the same certs. + local('scripts/generate_haproxy_certificate.sh viewfinder.co %s viewfinder.pem' % vf_passphrase) + run('mkdir -p ~/conf') + run('rm -f ~/conf/viewfinder.pem') + put('viewfinder.pem', '~/conf/viewfinder.pem') + run('chmod 400 ~/conf/viewfinder.pem') + + # Remove local file. + local('rm -f viewfinder.pem') + + # Install the config files. + fprint('Pushing haproxy configs') + assert env.nodetype, 'no nodetype specified' + run('ln -f -s ~/viewfinder/scripts/haproxy.conf ~/conf/haproxy.conf') + run('ln -f -s ~/viewfinder/scripts/haproxy.redirect.%s.conf ~/conf/haproxy.redirect.conf' % env.nodetype.lower()) + + +def setup_instance(zone, name, existing_instance_id=None): + if not existing_instance_id: + region_zones = ec2_utils.GetELBZones(env.region, node_types=[env.nodetype]) + assert zone, 'Availability zone not specified, available zones are: %s' % ' '.join(region_zones) + + user_data = load_passphrase_from_file() + instance_id = ec2_utils.RunInstance(env.region, BASE_AMI, 'wwwkey', kInstanceType, + availability_zone=zone, user_data=user_data) + fprint('Launched new instance: %s' % instance_id) + else: + instance_id = existing_instance_id + fprint('Resuming setup of instance %s' % instance_id) + + fprint('Adding tags NodeType=%s and Name=%s to instance %s' % (env.nodetype, name, instance_id)) + ec2_utils.CreateTag(env.region, instance_id, 'NodeType', env.nodetype) + ec2_utils.CreateTag(env.region, instance_id, 'Name', name) + + for i in range(60): + match = ec2_utils.GetInstance(env.region, instance_id) + if match is None: + fprint('Instance %s does not exist yet; waiting.' % instance_id) + elif match.state != 'running': + fprint('Instance %s in state %s; waiting.' % (instance_id, match.state)) + else: + break + time.sleep(2) + else: + fprint('Timed out waiting for instance: %s' % instance_id) + raise Exception("timeout") + assert match is not None and match.state == 'running' + instance_hostname = match.public_dns_name + fprint('Instance %s in state "running". Public DNS: %s' % (instance_id, instance_hostname)) + with settings(host_string=instance_hostname): + for i in range(60): + try: + run("true") + break + except NetworkError: + fprint('Waiting for instance to be sshable: %s' % instance_id) + # don't retry too aggressively, it looks like we get blocked by a + # firewall for too many failed attempts + time.sleep(3) + else: + fprint('timed out waiting for sshability') + raise Exception("timeout") + + # Install required packages. + yum_install() + return instance_id, instance_hostname + + +@task +def drain(): + """Drain nodes of a given type. + This removes the instance from the region load balancers for this instance type (STAGING or PROD). + """ + ami = get_ami_metadata() + instance_id = ami['instance-id'] + ec2_utils.RemoveELBInstance(env.region, instance_id, env.nodetype) + fprint('Removed instance %s from %s load balancers' % (instance_id, env.nodetype)) + + +@task +def undrain(): + """Undrain nodes of a given type. + This adds the instance from the region load balancers for this instance type (STAGING or PROD). + After addition, we query the load balancers until the instance health is InService. + """ + ami = get_ami_metadata() + instance_id = ami['instance-id'] + + fprint('Waiting for healthy backend') + num_healthz_ok = 0 + for i in range(60): + if get_healthz(): + num_healthz_ok += 1 + if num_healthz_ok >= 3: + break + else: + num_healthz_ok = 0 + time.sleep(2) + if num_healthz_ok < 3: + raise Exception('healthz timeout') + + ec2_utils.AddELBInstance(env.region, instance_id, env.nodetype) + fprint('Added instance %s to %s load balancers' % (instance_id, env.nodetype)) + for i in range(60): + health = ec2_utils.GetELBInstanceHealth(env.region, instance_id, node_types=[env.nodetype]) + if health is None: + fprint('No load balancer health information for instance %s; waiting.' % instance_id) + elif health == 'InService': + fprint('Load balancer health for instance %s is InService.' % instance_id) + return + else: + fprint('Load balancer health information for instance %s is %s; waiting.' % (instance_id, health)) + time.sleep(2) + raise Exception('timeout') + + +def check_min_healthy_instances(min_healthy): + """Lookup the number of instances by ELB state and assert if the minimum required is not met.""" + healthy = ec2_utils.GetELBInstancesByHealth(env.region, node_types=[env.nodetype]) + num_healthy = len(healthy['InService']) + assert num_healthy >= min_healthy, 'Not enough backends with healthy ELB status (%d vs %d)' % \ + (num_healthy, min_healthy) + +@task +def create_instance(nodetype, name, zone, existing_instance_id=None): + """Create a new instance. Specify NodeType,Name,AvailabilityZone,[id_to_resume].""" + env.nodetype = nodetype + + # Names must be unique across all node types. + named_instances = ec2_utils.ListInstances(env.region, names=[name]) + if named_instances: + assert len(named_instances) == 1, 'Multiple instances found with name %s' % name + prev_id = named_instances[0].id + assert existing_instance_id is not None and existing_instance_id == prev_id, \ + 'Name %s already in use by instance %s' % (name, prev_id) + assert name.startswith(nodetype), 'Instance name must start with %s' % nodetype + + instance_id, instance_hostname = setup_instance(zone, name, existing_instance_id=existing_instance_id) + with settings(host_string=instance_hostname): + deploy(new_instance=True) + + +@task +def destroy_instance(nodetype, instance_id): + """Stop and terminate an instance. Specify NodeType and InstanceID.""" + env.nodetype = nodetype + instance = ec2_utils.GetInstance(env.region, instance_id) + assert instance, 'Instance %s not found' % instance_id + with settings(host_string=instance.public_dns_name): + if instance.state == 'running': + check_min_healthy_instances(3) + drain() + stop() + fprint('Terminating instance %s' % instance_id) + ec2_utils.TerminateInstance(env.region, instance_id) + + +@task +def restart(): + """Restart supervisord and its managed jobs.""" + fprint('Restarting supervisord') + sudo('cp ~ec2-user/viewfinder/scripts/supervisord.d /etc/init.d/supervisord') + sudo('/etc/init.d/supervisord restart') + + +@task +def stop(): + """Stop supervisord and its managed jobs.""" + # TODO(marc): we should eventually use supervisordctl, but sending SIGTERM shuts it down properly for now. + fprint('Stopping supervisord') + # If we attempt a "deploy" with a new instance that hasn't been setup yet, we'll have no supervisord script to copy. + with settings(warn_only=True): + # We copy it since this may be the first call to supervisord. + # TODO(marc): remove 'cp' once supervisord init script is installed everywhere. + sudo('cp ~ec2-user/viewfinder/scripts/supervisord.d /etc/init.d/supervisord') + sudo('/etc/init.d/supervisord stop') + + +@task +def drainrestart(): + """Drain and restart nodes.""" + check_min_healthy_instances(2) + drain() + # Stop first to make sure we no longer use the viewfinder init scripts. + stop() + restart() + undrain() + + +@task +def deploy(new_instance=False): + """Deploy latest environment and code and restart backends.""" + # Run yum update/install first. We may have new dependencies. + yum_install() + # Push and build haproxy. + haproxy_install() + # Stage code, environment, and crontab. + virtualenv_install() + code_install() + install_crontab() + + if not new_instance: + # Remove backend from load balancer and stop. This would fail on non-running instances. + drain() + stop() + + # Flip symlinks. + virtualenv_activate() + code_activate() + + # Restart backend and re-add to load balancer. + restart() + undrain() + + +@task +def status(): + """Overall production status.""" + cl_timestamps = defaultdict(str) + def _ResolveCLDate(rev): + if rev == '??' or rev in cl_timestamps.keys(): + return + ts = hg_revision_timestamp(rev) + if ts is not None: + cl_timestamps[rev] = util.TimestampUTCToISO8601(ts) + + env_rev = latest_requirements_revision() + code_rev = hg_revision() + _ResolveCLDate(env_rev) + _ResolveCLDate(code_rev) + print '=' * 80 + print 'Local environment:' + print ' Env revision: %s (%s)' % (env_rev, cl_timestamps.get(env_rev, '??')) + print ' Code revision: %s (%s)' % (code_rev, cl_timestamps.get(code_rev, '??')) + + for nodetype in ec2_utils.kValidNodeTypes: + elbs = ec2_utils.GetLoadBalancers(env.region, [nodetype]) + assert len(elbs) == 1, 'Need exactly one %s load balancer in %s' % (nodetype, env.region) + + elb = elbs[0] + instances = ec2_utils.ListInstances(env.region, node_types=[nodetype]) + + elb_zones = {z:0 for z in elb.availability_zones} + elb_health = {h.instance_id: h.state for h in elb.get_instance_health()} + + for i in instances: + id = i.id + if i.state != 'running': + continue + zone = i.placement + if zone in elb_zones.keys(): + elb_zones[zone] += 1 + if id in elb_health: + setattr(i, '_elb_health', elb_health[id]) + with settings(host_string=i.public_dns_name): + setattr(i, '_env_rev', active_env() or '??') + setattr(i, '_code_rev', active_code() or '??') + _ResolveCLDate(i._env_rev) + _ResolveCLDate(i._code_rev) + + print '\n%s' % ('=' * 80) + print '%s ELB: %s' % (nodetype, elb.name) + print ' # Running instances by ELB zone:' + zone_str = ', '.join(['%s: %s' % (k, v) for k, v in elb_zones.iteritems()]) + print ' %s' % zone_str + + print '' + print '%s instances: %d' % (nodetype, len(instances)) + print ' # %-8s %-12s %-13s %-10s %-12s %-13s %-10s %-13s %-10s' % \ + ('ID', 'Name', 'State', 'Zone', 'ELB state', 'Active env', 'Env date', 'Active code', 'Code date') + for i in instances: + env_rev = getattr(i, '_env_rev', '') + code_rev = getattr(i, '_code_rev', '') + print ' %-10s %-12s %-13s %-10s %-12s %-13s %-10s %-13s %-10s' % (i.id, + i.tags.get('Name', ''), + i.state, + i.placement, + getattr(i, '_elb_health', ''), + env_rev, + cl_timestamps[env_rev], + code_rev, + cl_timestamps[code_rev]) + + if instances and code_rev and cl_timestamps.get(code_rev): + # If the deployed revision exists locally, create a bookmark to it (one per nodetype). + # This allows queries like these (some aliases are defined in viewfinder.hgrc) + # hg log -r ::.-::deployed_staging + # hg log -r ::deployed_staging-::deployed_prod + local("hg bookmark -f -r %s deployed_%s" % (code_rev, nodetype.lower())) + + +@task +def cleanup(): + """Cleanup old env and code.""" + assert env.host_string + + # Search for active env. + active_env_rev = active_env() + assert active_env_rev, 'No active env, this could be a problem; aborting.' + active_env_date = hg_revision_timestamp(active_env_rev) + assert active_env_date, 'Could not determine timestamp for active env revision %s; aborting.' % active_env_rev + + fprint('Current active environment is revision %s (%s)' % + (active_env_rev, util.TimestampUTCToISO8601(active_env_date))) + + # Search for, and iterate over, all environments. + installed_env_revs = run('ls -d ~/env.*') + for r in installed_env_revs.split(): + if not r.strip(): + continue + rev = get_file_suffix('env', r) + if not rev: + continue + if rev == active_env_rev: + continue + ts = hg_revision_timestamp(rev) + if not ts: + continue + if ts >= active_env_date: + fprint('Env revision %s (%s) newer than active env revision %s (%s); skipping.' % + (rev, util.TimestampUTCToISO8601(ts), active_env_rev, util.TimestampUTCToISO8601(active_env_date))) + continue + + answer = fprompt('Delete unused environment revision %s (%s)?' % (rev, util.TimestampUTCToISO8601(ts)), + default='N', validate='[yYnN]') + if answer == 'n' or answer == 'N': + continue + run('rm -r -f env.%s prod-requirements.txt.%s python-package.%s' % (rev, rev, rev)) + fprint('Deleted environment revision %s (%s)' % (rev, util.TimestampUTCToISO8601(ts))) + + # Search for active code. + active_code_rev = active_code() + assert active_code_rev, 'No active code, this could be a problem; aborting.' + active_code_date = hg_revision_timestamp(active_code_rev) + assert active_code_date, 'Could not determine timestamp for active code revision %s; aborting.' % active_code_rev + + fprint('Current active code is revision %s (%s)' % + (active_code_rev, util.TimestampUTCToISO8601(active_code_date))) + + # Search for, and iterate over, all code. + installed_code_revs = run('ls -d ~/viewfinder.*') + for r in installed_code_revs.split(): + if not r.strip(): + continue + rev = get_file_suffix('viewfinder', r) + if not rev: + continue + if rev == active_code_rev: + continue + ts = hg_revision_timestamp(rev) + if not ts: + continue + if ts >= active_code_date: + fprint('Code revision %s (%s) newer than active code revision %s (%s); skipping.' % + (rev, util.TimestampUTCToISO8601(ts), active_code_rev, util.TimestampUTCToISO8601(active_code_date))) + continue + + answer = fprompt('Delete unused code revision %s (%s)?' % (rev, util.TimestampUTCToISO8601(ts)), + default='N', validate='[yYnN]') + if answer == 'n' or answer == 'N': + continue + run('rm -r -f viewfinder.%s' % rev) + fprint('Deleted code revision %s (%s)' % (rev, util.TimestampUTCToISO8601(ts))) diff --git a/marketing/.hgignore b/marketing/.hgignore new file mode 100644 index 0000000..2c9154d --- /dev/null +++ b/marketing/.hgignore @@ -0,0 +1,2 @@ +syntax: glob +*.pyc diff --git a/marketing/README b/marketing/README new file mode 100644 index 0000000..fdee1e0 --- /dev/null +++ b/marketing/README @@ -0,0 +1,8 @@ +Viewfinder marketing website +============================ + +To work on the marketing site on its own, run server.py and +visit http://localhost:8443 in your browser. + +In the full viewfinder repo the marketing site is loaded as a subrepo +and served from the main server via symlinks. \ No newline at end of file diff --git a/marketing/resources/static/marketing/css/global.css b/marketing/resources/static/marketing/css/global.css new file mode 100644 index 0000000..d1fe6c5 --- /dev/null +++ b/marketing/resources/static/marketing/css/global.css @@ -0,0 +1,1428 @@ +@charset "UTF-8"; +/*! normalize.css v2.1.2 | MIT License | git.io/normalize */ +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +nav, +section, +summary { + display: block; +} +audio, +canvas, +video { + display: inline-block; +} +audio:not([controls]) { + display: none; + height: 0; +} +[hidden] { + display: none; +} +html { + font-family: sans-serif; + -ms-text-size-adjust: 100%; + -webkit-text-size-adjust: 100%; +} +body { + margin: 0; +} +a:focus { + outline: thin dotted; +} +a:active, +a:hover { + outline: 0; +} +h1 { + font-size: 2em; + margin: 0.67em 0; +} +abbr[title] { + border-bottom: 1px dotted; +} +b, +strong { + font-weight: bold; +} +dfn { + font-style: italic; +} +hr { + -moz-box-sizing: content-box; + box-sizing: content-box; + height: 0; +} +mark { + background: #ff0; + color: #000000; +} +code, +kbd, +pre, +samp { + font-family: monospace,serif; + font-size: 1em; +} +pre { + white-space: pre-wrap; +} +q { + quotes: "\201C" "\201D" "\2018" "\2019"; +} +small { + font-size: 80%; +} +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} +sup { + top: -0.5em; +} +sub { + bottom: -0.25em; +} +img { + border: 0; +} +svg:not(:root) { + overflow: hidden; +} +figure { + margin: 0; +} +fieldset { + border: 1px solid silver; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} +legend { + border: 0; + padding: 0; +} +button, +input, +select, +textarea { + font-family: inherit; + font-size: 100%; + margin: 0; +} +button, +input { + line-height: normal; +} +button, +select { + text-transform: none; +} +button, +html input[type="button"], +input[type="reset"], +input[type="submit"] { + -webkit-appearance: button; + cursor: pointer; +} +button[disabled], +html input[disabled] { + cursor: default; +} +input[type="checkbox"], +input[type="radio"] { + box-sizing: border-box; + padding: 0; +} +input[type="search"] { + -webkit-appearance: textfield; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; +} +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} +button::-moz-focus-inner, +input::-moz-focus-inner { + border: 0; + padding: 0; +} +textarea { + overflow: auto; + vertical-align: top; +} +table { + border-collapse: collapse; + border-spacing: 0; +} +* { + outline: none; + -moz-box-sizing: border-box; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} +html, +body { + margin: 0; + padding: 0; +} +a { + border: none; +} +input, +textarea { + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; +} +.noselect { + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -o-user-select: none; + user-select: none; +} +.clear { + clear: both; +} +#debug { + position: fixed; + padding: 3px; + bottom: 0; + right: 0; + background: #333; + color: #fff; + display: none; +} +h2 { + color: #ff9625; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 42px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + line-height: 46px; + text-align: left; + letter-spacing: -1px; + line-height: 44px; +} +p { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: left; + padding: 0; + line-height: 22px; + text-decoration: none; +} +#animate_ref { + position: fixed; + bottom: 0; + left: 0; + width: 1px; + height: 1px; + opacity: 0; +} +.content_panel { + margin: 0 auto; + width: 100%; + max-width: 1400px; + text-align: center; + position: relative; + float: left; +} +.full_panel { + margin: 0 auto; + width: 100%; + text-align: center; + position: relative; + float: left; +} +.download { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: center; + border: none; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + text-transform: uppercase; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + /*padding:15px 25px 12px;*/ + text-decoration: none; + line-height: 14px; + display: block; + letter-spacing: 2px; + padding: 15px 25px; + background: #ffb34b; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2ZmYjM0YiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmZhNTJhIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmODcyMCIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffb34b), color-stop(2%, #ffa52a), color-stop(100%, #ff8720)); + background: -webkit-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -o-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -ms-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: linear-gradient(to bottom, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffb34b', endColorstr='#ff8720', GradientType=0); +} +.download.large { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 18px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + padding: 20px 40px; + letter-spacing: 4px; +} +.download:hover { + background: #ffa645; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2ZmYTY0NSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmY5NjI1IiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmOTYyNSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffa645), color-stop(2%, #ff9625), color-stop(100%, #ff9625)); + background: -webkit-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -o-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -ms-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: linear-gradient(to bottom, #ffa645 0%, #ff9625 2%, #ff9625 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffa645', endColorstr='#ff9625', GradientType=0); +} +.download:active { + background: #b25e16; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2IyNWUxNiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmY4NzIwIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmYTUyYSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #b25e16), color-stop(2%, #ff8720), color-stop(100%, #ffa52a)); + background: -webkit-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -o-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -ms-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: linear-gradient(to bottom, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b25e16', endColorstr='#ffa52a', GradientType=0); +} +.watch_demo { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: center; + border: none; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + text-transform: uppercase; + margin: 20px 0 0 0; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 15px 25px 12px; + text-decoration: none; + line-height: 14px; + clear: both; + float: left; + background: #b9b8b8; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2I5YjhiOCIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjYWRhYWFhIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iIzkxOGU4ZSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #b9b8b8), color-stop(2%, #adaaaa), color-stop(100%, #918e8e)); + background: -webkit-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -o-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -ms-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: linear-gradient(to bottom, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b9b8b8', endColorstr='#918e8e', GradientType=0); +} +.watch_demo .icon { + margin: 0 0 -2px 5px; +} +.watch_demo:hover { + background: #adabab; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2FkYWJhYiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjOWY5YzljIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iIzlmOWM5YyIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #adabab), color-stop(2%, #9f9c9c), color-stop(100%, #9f9c9c)); + background: -webkit-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -o-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -ms-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: linear-gradient(to bottom, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#adabab', endColorstr='#9f9c9c', GradientType=0); +} +.watch_demo:active { + background: #656262; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iIzY1NjI2MiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjOTE4ZThlIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2FkYWFhYSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #656262), color-stop(2%, #918e8e), color-stop(100%, #adaaaa)); + background: -webkit-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -o-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -ms-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: linear-gradient(to bottom, #656262 0%, #918e8e 2%, #adaaaa 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#656262', endColorstr='#adaaaa', GradientType=0); +} +#top_nav { + min-height: 88px; + padding: 0; + border-bottom: 1px solid #D0CBCB; +} +#top_nav .content_frame { + max-width: 1400px; + width: 100%; + margin: 0 auto; +} +#top_nav h1 { + width: 306px; + height: 66px; + float: left; + margin: 12px 0 0 10%; +} +#top_nav h1 a { + width: 100%; + height: 100%; + float: left; + display: block; + background: url('/static/marketing/images/Viewfinder-Logo-Beta@2x.png') -15px 0 no-repeat; + background-size: 306px 66px; + width: 306px; + height: 66px; +} +#top_nav ul { + float: right; + margin: 27px 10% 0 0; +} +#top_nav ul li { + float: right; + list-style: none; + margin: 0 0 0 65px; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 6px 10px; +} +#top_nav ul li a { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + text-decoration: none; + color: #3F3E3E; +} +#top_nav ul li:last-child { + margin-left: 0; +} +#top_nav ul li:hover { + background: #ff9625; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; +} +#top_nav ul li:hover a { + color: #fff; +} +#top_nav .menu { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + text-decoration: none; + color: #fff; + background: #3F3E3E; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; + float: right; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 8px 13px 5px; + margin: 27px 10% 0 0; + display: none; +} +#top_nav .menu.open { + background: #ECE9E9; + color: #C73926; +} +#top_nav .watch_demo_link { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #3F3E3E; + text-align: center; + text-transform: uppercase; + float: right; + margin: 35px 10% 0 0; + text-decoration: none; +} +#top_nav .watch_demo_link .icon { + margin: 0 0 -2px 5px; +} +#home_hero { + height: 570px; + overflow: hidden; + background: #000; + text-align: center; + position: relative; +} +#home_hero .bkgd_img, +#home_hero .bkgd_img_off { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-position: center 0; + background-repeat: no-repeat; + z-index: 0; +} +#home_hero .bkgd_img_off { + display: none; +} +#home_hero .content_frame { + position: relative; + z-index: 1; + margin: 0 auto; + max-width: 1400px; + padding: 0 10%; + width: 100%; + display: inline-block; +} +#home_hero h2 { + color: #fff; + float: left; + position: relative; + z-index: 1; + text-align: left; + width: 400px; + margin: 230px 0 0 0; +} +#home_hero .download { + position: relative; + z-index: 1; + clear: both; + float: left; + margin: 30px 0 0 0; +} +#tour_panel { + padding: 130px 8% 130px; +} +#tour_panel .content_frame { + border: 15px solid #FAF7F7; + width: 100%; + height: auto; + max-width: 1000px; + margin: 0 auto; +} +#tour_panel .content_frame .phone { + position: relative; + float: right; + width: 382px; + height: 732px; + margin: -95px 0 -95px 40px; +} +#tour_points { + float: right; + text-align: left; + margin: 132px 0px 0 85px; + width: 330px; +} +#tour_points h2 { + margin: 0 0 22px 0px; + width: 100%; +} +#tour_points ul { + list-style: none; + margin: 0; + padding: 0; + width: 100%; +} +#tour_points ul li { + list-style: none; + margin: 0 0 20px 0; + padding: 0; + position: relative; +} +#tour_points ul li a { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #bcbcbc; + padding: 0; + line-height: 20px; + text-decoration: none; + position: relative; +} +#tour_points ul li .radio { + position: absolute; + top: 0; + left: -40px; + background: url(/static/marketing/images/radio_sprite@2x.png) -43px -44px no-repeat; + background-size: 80px 86px; + width: 21px; + height: 21px; +} +#tour_points ul li:hover a { + color: #ff9625; +} +#tour_points ul li:hover .radio { + background-position: -43px -66px; +} +#tour_points ul li.current a { + color: #3F3E3E; +} +#tour_points ul li.current .radio { + background-position: -43px 0; +} +#tour_points ul li.current:hover a { + color: #ff9625; +} +#tour_points ul li.current:hover .radio { + background-position: -43px -22px; +} +#tour_points .radio_buttons_small { + display: none; + float: left; + width: auto; +} +#tour_points .radio_buttons_small li { + width: 11px; + height: 10px; + float: left; + margin-right: 10px; +} +#tour_points .radio_buttons_small li .radio { + position: relative; + top: 0; + left: 0; + float: left; + background: url(/static/marketing/images/radio_sprite@2x.png) -69px -49px no-repeat; + width: 11px; + height: 10px; + background-size: 80px 86px; +} +#tour_points .radio_buttons_small li:hover a { + color: #ff9625; +} +#tour_points .radio_buttons_small li:hover .radio { + background-position: -69px -71px; +} +#tour_points .radio_buttons_small li.current a { + color: #3F3E3E; +} +#tour_points .radio_buttons_small li.current .radio { + background-position: -69px -5px; +} +#tour_points .radio_buttons_small li.current:hover a { + color: #ff9625; +} +#tour_points .radio_buttons_small li.current:hover .radio { + background-position: -69px -27px; +} +#tour_points .radio_buttons_small li:last-child { + margin-right: 0; +} +#tour_video video { + float: left; +} +#tour_copy { + padding: 0; + height: 100%; + display: inline-block; + margin: 0 auto; +} +#tour_video_holder { + background-position: 0 0; + background-repeat: no-repeat; + width: 268px; + height: 476px; + position: absolute; + top: 128px; + left: 57px; + overflow: hidden; +} +#tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg'); + background-size: 268px 476px; + } +} +#tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg'); + background-size: 268px 476px; + } +} +#tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg'); + background-size: 268px 476px; + } +} +#dial_panel { + background: #222; + overflow: hidden; + height: auto; +} +#dial_panel .bkgd_img { + background-image: url('/static/marketing/images/Home-02-iPhone-Dial.jpg'); + background-position: right top; + background-repeat: no-repeat; + height: 480px; + margin: 0 auto; + width: 100%; + min-width: 900px; + z-index: 0; +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #dial_panel .bkgd_img { + background-image: url('/static/marketing/images/Home-02-iPhone-Dial@2x.jpg'); + background-size: 1200px 836px; + } +} +#dial_panel .content_frame_abs { + position: absolute; + z-index: 1; + width: 100%; + top: 0; + left: 0; +} +#dial_panel .content_frame { + position: relative; + z-index: 1; + padding: 0 10%; + width: 100%; + margin: 0 auto; + max-width: 1400px; + display: inline-block; +} +#dial_panel .content_frame:hover { + cursor: pointer; +} +#dial_panel h2 { + color: #fff; + float: left; + position: relative; + z-index: 1; + width: 300px; + margin: 160px 0 0 0; +} +#dial_panel p { + color: #fff; + clear: both; + float: left; + position: relative; + width: 375px; +} +#dial_panel .bkgd_video { + width: 100%; + position: absolute; + top: 0; + left: 0; +} +#dial_panel .close { + position: absolute; + top: 15px; + left: 10px; + width: 33px; + height: 33px; + z-index: 10; + display: none; +} +#dial_panel.expanded .content_frame:hover { + cursor: default; +} +#dial_panel.has_video { + background: #413e40; +} +#dial_panel.has_video .bkgd_img { + background: #413e40; +} +#dial_video_holder { + float: left; + display: none; +} +#dial_bkgd_video { + float: right; +} +#preserve_panel { + padding: 130px 8%; +} +#preserve_panel .content_panel { + position: absolute; + z-index: 1; + top: 0; + left: 0; + padding: 0 10%; +} +#preserve_panel .content_frame { + border: 15px solid #FAF7F7; + width: 100%; + height: 572px; + text-align: center; + max-width: 1000px; + margin: 0 auto; +} +#preserve_panel .content_frame h2 { + color: #ff9625; + float: left; + position: relative; + z-index: 1; + width: 300px; + margin: 120px 0 0 0; +} +#preserve_panel .content_frame p { + color: #3F3E3E; + clear: both; + float: left; + position: relative; + width: 300px; +} +#preserve_panel .content_frame .download { + position: relative; + z-index: 1; + clear: both; + float: left; + margin: 30px 0 0 0; +} +#preserve_copy { + padding: 0 0 0 405px; + height: 100%; + background: url(/static/marketing/images/viewFinder_amber-small400.png) 0 100px no-repeat; + display: inline-block; + margin: 0 auto; +} +#download_panel { + padding: 70px 0 80px 0; + border-top: 1px solid #CFCBCB; +} +#download_panel h2 { + width: 100%; + text-align: center; + color: #3F3E3E; + clear: both; + float: left; + margin-bottom: 44px; +} +#download_panel .download_holder, +#download_panel .take_tour_holder, +#download_panel .watch_video_holder { + width: 100%; + float: left; + text-align: center; + margin: 0px 0 20px; +} +#download_panel .download { + display: inline-block; + margin: 0 auto; +} +#download_panel .watch_video_holder { + margin: 0 0 20px 0; +} +#download_panel .watch_video_holder .watch_demo { + float: none; + display: inline-block; + margin: 0 auto; +} +#download_panel .take_tour_holder { + margin: 0; +} +#download_panel .take_tour { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #ff9625; + text-decoration: none; + text-transform: uppercase; + letter-spacing: 2px; +} +#download_panel .take_tour:hover { + color: #ff8720; +} +#footer { + width: 100%; + clear: both; + float: left; + background: #FAF7F7; + padding: 70px 10% 40px; +} +#footer .content_frame { + max-width: 1400px; + width: 100%; + margin: 0 auto; +} +#footer ul { + float: left; + margin-bottom: 60px; + padding: 0; +} +#footer ul li { + float: left; + list-style: none; + margin: 0 60px 0 0; + padding: 0; +} +#footer ul li a { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + text-decoration: none; + color: #3F3E3E; +} +#footer ul li a:hover { + color: #ff9625; +} +#footer ul li.right { + float: right; +} +#footer ul li:last-child { + margin-right: 0; +} +#footer ul.share_nav { + float: right; +} +#footer ul.share_nav li { + margin: 0 0 0 60px; +} +#footer ul.share_nav li:first-child { + margin-left: 0; +} +#footer .footer_note { + clear: both; + text-align: center; +} +#footer .footer_note .note_partial { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #9F9C9C; + text-align: left; + display: inline; +} +#footer .footer_note a { + color: #3F3E3E; + text-decoration: none; +} +#footer .footer_note a:hover { + color: #ff9625; +} +/* + * Secondary Page Styles + * + */ +#secondary-container { + color: #3F3E3E; +} +#secondary-container .content_frame { + max-width: 1400px; + margin: 0 auto; + text-align: left; + padding: 0 10% 100px; +} +#secondary-container h2 { + color: #3F3E3E; + max-width: 600px; + margin-top: 50px; +} +#secondary-container .last-modified { + color: #9F9C9C; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + max-width: 600px; +} +#secondary-container .separator { + max-width: 600px; + height: 2px; + margin: 15px 0 45px; + border-bottom: 1px solid #A09C9C; +} +#secondary-container p { + color: #3F3E3E; + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + max-width: 600px; + line-height: 21px; +} +#secondary-container a { + color: #3F3E3E; +} +#secondary-container h3 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 22px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + max-width: 600px; + margin: 40px 0 20px; + line-height: 26px; +} +#secondary-container h4 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + max-width: 600px; + margin: 40px 0 10px; +} +#secondary-container ul { + width: 100%; + max-width: 600px; +} +#secondary-container ul li { + color: #3F3E3E; + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + max-width: 600px; + width: 100%; + line-height: 21px; + margin-bottom: 5px; +} +body.faq .faq-toc { + padding: 0; + margin: 0; +} +body.faq .faq-toc li { + margin: 0; + padding: 0; + list-style: none; +} +body.faq .faq-toc li a { + text-decoration: none; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; +} +body.faq .faq-list { + padding: 0; + margin: 0; +} +body.faq .faq-item { + list-style: none; + padding: 0; + margin: 0; +} +@media screen and (max-width: 1100px) { + h2 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 26px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + font-weight: 500; + line-height: 30px; + letter-spacing: 0; + } + #home_hero { + height: 352px; + } + #home_hero h2 { + width: 250px; + margin: 110px 0 0 0; + } + #home_hero .download { + margin: 20px 0 0 0; + } + #tour_panel { + padding: 30px 8% 10px; + } + #tour_panel .content_frame { + border: none; + width: 100%; + } + #tour_panel .content_frame .phone { + width: 282px; + height: 542px; + margin: 0; + } + #tour_points { + margin: 120px 30px 0 20px; + width: 240px; + } + #tour_points ul li .radio { + display: none; + } + #tour_points .radio_buttons_small { + display: inline; + } + #tour_points .radio_buttons_small li .radio { + display: inline; + } + #tour_video_holder { + width: 201px; + height: 357px; + top: 91px; + left: 40px; + } + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg'); + background-size: 201px 357px; + } + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg'); + background-size: 201px 357px; + } + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg'); + background-size: 201px 357px; + } + #dial_panel { + height: auto; + } + #dial_panel .bkgd_img { + background: url('/static/marketing/images/Home-02-iPhone-Dial.jpg') right top no-repeat; + background-size: 535px 373px; + height: 373px; + } + #dial_panel h2 { + width: 330px; + } + #dial_panel p { + width: 330px; + } + #preserve_panel { + padding: 0 8% 115px; + overflow: hidden; + } + #preserve_panel .content_frame { + border: none; + height: auto; + } + #preserve_panel .content_frame h2 { + width: 235px; + margin: 115px 0 0 0; + } + #preserve_panel .content_frame p { + width: 235px; + } + #preserve_panel .content_frame .download { + margin: 20px 0 0 0; + } + #preserve_copy { + padding: 0 30px 0 360px; + background-position: 0 70px; + } + #footer { + padding: 30px 10% 40px; + } + #footer ul { + float: left; + margin: 0; + width: 100%; + text-align: center; + } + #footer ul li { + margin: 0 30px 0 0; + float: none; + display: inline; + } + #footer ul.share_nav { + clear: both; + width: 100%; + margin: 20px auto 30px; + text-align: center; + } + #footer ul.share_nav li { + margin: 0 0 0 30px; + display: inline; + float: none; + } + #footer .footer_note .crafted { + float: left; + width: 100%; + text-align: center; + display: block; + margin-bottom: 5px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1000px) { + #top_nav ul li { + margin: 0 0 0 35px; + } + #top_nav ul li:last-child { + margin-left: 35px; + } +} +@media screen and (max-width: 900px) { + #top_nav h1 { + width: 213px; + height: 35px; + float: left; + margin-left: 5%; + } + #top_nav ul { + margin: 27px 5% 0 0; + } + #top_nav ul li { + margin: 0 0 0 25px; + } + #top_nav ul li:last-child { + margin-left: 25px; + } + #tour_panel { + padding-left: 2%; + padding-right: 2%; + } + #secondary-container .content_frame { + padding: 0 5% 50px; + } +} +@media screen and (max-width: 800px) { + #top_nav .nav_holder { + clear: both; + float: left; + width: 100%; + margin: 27px 0 0; + height: 0; + overflow: hidden; + } + #top_nav ul { + clear: both; + float: left; + margin: 0; + padding: 35px 0; + width: 100%; + text-align: center; + border-top: 1px solid #CFCBCB; + } + #top_nav ul li { + float: none; + display: inline; + margin: 0 0 0 65px; + } + #top_nav ul li.home { + display: none; + } + #top_nav ul li:first-child { + margin-left: 0; + } + #top_nav ul li:last-child { + margin-left: 65px; + } + #top_nav .menu { + display: inline; + margin-right: 5%; + } +} +@media screen and (max-width: 600px) { + #top_nav { + min-height: 66px; + } + #top_nav h1 { + width: 36px; + height: 36px; + margin-top: 15px; + } + #top_nav h1 a { + background: url('/static/marketing/images/logo_icon@2x.png') 0 0 no-repeat; + background-size: 34px 34px; + background-position: 0 0; + background-repeat: no-repeat; + } + #top_nav ul li { + margin: 0 0 0 30px; + } + #top_nav .menu { + margin-top: 17px; + } + #top_nav .nav_holder { + margin-top: 17px; + } + #top_nav .watch_demo_link { + margin-top: 25px; + } + #home_hero h2 { + width: 100%; + margin-top: 200px; + text-align: center; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 20px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + font-weight: 500; + line-height: 24px; + } + #home_hero .download { + margin: 20px auto 0; + float: none; + display: inline-block; + } + #tour_panel .content_frame { + text-align: center; + } + #tour_panel .content_frame .phone { + margin: 0 auto; + display: inline-block; + float: none; + } + #tour_points { + margin: 0; + width: 100%; + text-align: center; + } + #tour_points h2 { + width: 100%; + text-align: center; + } + #tour_points ul.points { + margin-bottom: 20px; + text-align: center; + float: left; + } + #tour_points ul.points li { + display: none; + margin: 0 auto; + text-align: center; + width: 70%; + margin-left: 15%; + float: left; + min-height: 40px; + } + #tour_points ul.points li.current { + display: inline; + } + #tour_points .radio_buttons_small { + display: inline-block; + float: none; + } + #dial_panel { + height: auto; + } + #dial_panel .bkgd_img { + /*background:url(/static/marketing/images/dial_bkgd_small.png) right top no-repeat; */ + background: none; + height: 285px; + min-width: 300px; + } + #dial_panel h2 { + margin-top: 66px; + width: 100%; + text-align: center; + } + #dial_panel p { + width: 100%; + text-align: center; + } + #dial_panel .watch_demo { + float: none; + display: inline-block; + } + #dial_bkgd_video { + display: none; + } + #preserve_panel { + padding: 0 0; + overflow: hidden; + } + #preserve_panel .content_frame { + border: none; + width: 100%; + margin: 0 auto; + } + #preserve_panel .content_frame h2 { + width: 84%; + text-align: center; + margin: 30px 0 0 8%; + } + #preserve_panel .content_frame p { + width: 84%; + text-align: center; + margin-left: auto; + margin-right: auto; + display: inline-block; + float: none; + } + #preserve_panel .content_frame .download { + margin: 10px auto 0; + float: none; + display: inline-block; + } + #preserve_copy { + padding: 0 0 280px 0; + background-position: center bottom; + margin-bottom: 20px; + text-align: center; + } + #download_panel { + padding: 25px 0 40px 0; + } + #download_panel h2 { + width: 70%; + float: none; + display: inline-block; + margin: 20px auto 44px; + max-width: 230px; + } + #footer { + text-align: center; + } + #footer .footer_note .copyright, + #footer .footer_note .terms { + float: left; + width: 100%; + text-align: center; + display: block; + margin-bottom: 5px; + } + #footer ul.footer_nav { + width: auto; + float: none; + display: inline-block; + margin: 0 auto; + } + #footer ul.footer_nav li { + margin-bottom: 20px; + float: left; + } + #footer ul.footer_nav .journal { + margin-right: 0; + } +} +@media screen and (max-width: 480px) { + #dial_panel .bkgd_img { + height: 385px; + } +} diff --git a/marketing/resources/static/marketing/css/marketing.css b/marketing/resources/static/marketing/css/marketing.css new file mode 100644 index 0000000..77788c1 --- /dev/null +++ b/marketing/resources/static/marketing/css/marketing.css @@ -0,0 +1,2982 @@ +@charset "UTF-8"; +/** + * Division Of Resets / Boilerplate + * + */ +/*! normalize.css v2.1.2 | MIT License | git.io/normalize */ +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +nav, +section, +summary { + display: block; +} +audio, +canvas, +video { + display: inline-block; +} +audio:not([controls]) { + display: none; + height: 0; +} +[hidden] { + display: none; +} +html { + font-family: sans-serif; + -ms-text-size-adjust: 100%; + -webkit-text-size-adjust: 100%; +} +body { + margin: 0; +} +a:focus { + outline: thin dotted; +} +a:active, +a:hover { + outline: 0; +} +h1 { + font-size: 2em; + margin: 0.67em 0; +} +abbr[title] { + border-bottom: 1px dotted; +} +b, +strong { + font-weight: bold; +} +dfn { + font-style: italic; +} +hr { + -moz-box-sizing: content-box; + box-sizing: content-box; + height: 0; +} +mark { + background: #ff0; + color: #000000; +} +code, +kbd, +pre, +samp { + font-family: monospace,serif; + font-size: 1em; +} +pre { + white-space: pre-wrap; +} +q { + quotes: "\201C" "\201D" "\2018" "\2019"; +} +small { + font-size: 80%; +} +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} +sup { + top: -0.5em; +} +sub { + bottom: -0.25em; +} +img { + border: 0; +} +svg:not(:root) { + overflow: hidden; +} +figure { + margin: 0; +} +fieldset { + border: 1px solid silver; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} +legend { + border: 0; + padding: 0; +} +button, +input, +select, +textarea { + font-family: inherit; + font-size: 100%; + margin: 0; +} +button, +input { + line-height: normal; +} +button, +select { + text-transform: none; +} +button, +html input[type="button"], +input[type="reset"], +input[type="submit"] { + -webkit-appearance: button; + cursor: pointer; +} +button[disabled], +html input[disabled] { + cursor: default; +} +input[type="checkbox"], +input[type="radio"] { + box-sizing: border-box; + padding: 0; +} +input[type="search"] { + -webkit-appearance: textfield; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; +} +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} +button::-moz-focus-inner, +input::-moz-focus-inner { + border: 0; + padding: 0; +} +textarea { + overflow: auto; + vertical-align: top; +} +table { + border-collapse: collapse; + border-spacing: 0; +} +* { + outline: none; + -moz-box-sizing: border-box; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} +html, +body { + margin: 0; + padding: 0; +} +body { + overflow-y: scroll; +} +a { + border: none; +} +input, +textarea { + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; +} +.noselect { + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -o-user-select: none; + user-select: none; +} +.clear { + clear: both; +} +#debug { + position: fixed; + padding: 3px; + bottom: 0; + right: 0; + background: #333; + color: #fff; + display: none; + z-index: 99999; +} +#css_vp_wid { + display: none; + position: fixed; + bottom: 0; + left: 0; + width: 100%; +} +/** + * Viewfinder Global Styles + * + */ +h2 { + color: #ff9625; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 42px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + line-height: 46px; + text-align: left; + letter-spacing: -1px; + line-height: 44px; +} +p { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: left; + padding: 0; + line-height: 22px; + text-decoration: none; +} +#animate_ref { + position: fixed; + bottom: 0; + left: 0; + width: 1px; + height: 1px; + opacity: 0; +} +.content_panel { + margin: 0 auto; + width: 100%; + max-width: 1400px; + text-align: center; + position: relative; + float: left; +} +.full_panel { + margin: 0 auto; + width: 100%; + text-align: center; + position: relative; + float: left; +} +.download { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: center; + border: none; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + text-transform: uppercase; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + /*padding:15px 25px 12px;*/ + text-decoration: none; + line-height: 14px; + display: block; + letter-spacing: 2px; + padding: 15px 25px 12px; + background: #ffb34b; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2ZmYjM0YiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmZhNTJhIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmODcyMCIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffb34b), color-stop(2%, #ffa52a), color-stop(100%, #ff8720)); + background: -webkit-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -o-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: -ms-linear-gradient(top, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + background: linear-gradient(to bottom, #ffb34b 0%, #ffa52a 2%, #ff8720 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffb34b', endColorstr='#ff8720', GradientType=0); +} +.download.large { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 18px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + padding: 20px 40px; + letter-spacing: 4px; +} +.download:hover { + background: #ffa645; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2ZmYTY0NSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmY5NjI1IiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmOTYyNSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #ffa645), color-stop(2%, #ff9625), color-stop(100%, #ff9625)); + background: -webkit-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -o-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: -ms-linear-gradient(top, #ffa645 0%, #ff9625 2%, #ff9625 100%); + background: linear-gradient(to bottom, #ffa645 0%, #ff9625 2%, #ff9625 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffa645', endColorstr='#ff9625', GradientType=0); +} +.download:active { + background: #b25e16; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2IyNWUxNiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjZmY4NzIwIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2ZmYTUyYSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #b25e16), color-stop(2%, #ff8720), color-stop(100%, #ffa52a)); + background: -webkit-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -o-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: -ms-linear-gradient(top, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + background: linear-gradient(to bottom, #b25e16 0%, #ff8720 2%, #ffa52a 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b25e16', endColorstr='#ffa52a', GradientType=0); +} +.watch_demo { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #fff; + text-align: center; + border: none; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + text-transform: uppercase; + margin: 20px 0 0 0; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 12px 25px 13px; + text-decoration: none; + line-height: 14px; + clear: both; + float: left; + letter-spacing: 2px; + background: #b9b8b8; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2I5YjhiOCIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjYWRhYWFhIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iIzkxOGU4ZSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #b9b8b8), color-stop(2%, #adaaaa), color-stop(100%, #918e8e)); + background: -webkit-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -o-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: -ms-linear-gradient(top, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + background: linear-gradient(to bottom, #b9b8b8 0%, #adaaaa 2%, #918e8e 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#b9b8b8', endColorstr='#918e8e', GradientType=0); +} +.watch_demo .icon { + margin: 0 0 -3px 5px; +} +.watch_demo:hover { + background: #adabab; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iI2FkYWJhYiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjOWY5YzljIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iIzlmOWM5YyIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #adabab), color-stop(2%, #9f9c9c), color-stop(100%, #9f9c9c)); + background: -webkit-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -o-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: -ms-linear-gradient(top, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + background: linear-gradient(to bottom, #adabab 0%, #9f9c9c 2%, #9f9c9c 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#adabab', endColorstr='#9f9c9c', GradientType=0); +} +.watch_demo:active { + background: #656262; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iIzY1NjI2MiIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgICA8c3RvcCBvZmZzZXQ9IjIlIiBzdG9wLWNvbG9yPSIjOTE4ZThlIiBzdG9wLW9wYWNpdHk9IjEiLz4KICAgIDxzdG9wIG9mZnNldD0iMTAwJSIgc3RvcC1jb2xvcj0iI2FkYWFhYSIgc3RvcC1vcGFjaXR5PSIxIi8+CiAgPC9saW5lYXJHcmFkaWVudD4KICA8cmVjdCB4PSIwIiB5PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBmaWxsPSJ1cmwoI2dyYWQtdWNnZy1nZW5lcmF0ZWQpIiAvPgo8L3N2Zz4=); + background: -moz-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #656262), color-stop(2%, #918e8e), color-stop(100%, #adaaaa)); + background: -webkit-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -o-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: -ms-linear-gradient(top, #656262 0%, #918e8e 2%, #adaaaa 100%); + background: linear-gradient(to bottom, #656262 0%, #918e8e 2%, #adaaaa 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#656262', endColorstr='#adaaaa', GradientType=0); +} +#top_nav { + min-height: 88px; + padding: 0; + border-bottom: 1px solid #D0CBCB; +} +#top_nav .content_frame { + max-width: 1400px; + width: 100%; + margin: 0 auto; +} +#top_nav h1 { + width: 306px; + height: 66px; + float: left; + margin: 12px 0 0 10%; +} +#top_nav h1 a { + width: 100%; + height: 100%; + float: left; + display: block; + background: url('/static/marketing/images/Viewfinder-Logo-Beta@2x.png') -15px 0 no-repeat; + background-size: 306px 66px; + width: 306px; + height: 66px; +} +#top_nav ul { + float: right; + margin: 30px 10% 0 0; +} +#top_nav ul li { + float: right; + list-style: none; + margin: 0 0 0 65px; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 6px 10px; +} +#top_nav ul li a { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + text-decoration: none; + color: #3F3E3E; + letter-spacing: 2px; +} +#top_nav ul li.nav_watch_demo a { + background: none; +} +#top_nav ul li.nav_watch_demo .icon { + margin: 0 0 -2px 5px; +} +#top_nav ul li.nav_watch_demo .on { + display: none; +} +#top_nav ul li.nav_watch_demo:hover .on { + display: inline; +} +#top_nav ul li.nav_watch_demo:hover .off { + display: none; +} +#top_nav ul li:last-child { + margin-left: 0; +} +#top_nav ul li:hover, +#top_nav ul li.current { + background: #ff9625; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; +} +#top_nav ul li:hover a, +#top_nav ul li.current a { + color: #fff; +} +#top_nav .menu { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + text-decoration: none; + color: #fff; + background: #3F3E3E; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; + float: right; + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; + box-sizing: content-box; + padding: 8px 13px 5px; + margin: 27px 10% 0 0; + display: none; +} +#top_nav .menu.open { + background: #ECE9E9; + color: #C73926; +} +#download_panel { + padding: 70px 0 80px 0; + border-top: 1px solid #CFCBCB; + /* + .download_holder, .take_tour_holder, .watch_video_holder { width:100%; float:left; text-align:center; margin:0px 0 20px; } + .download { display:inline-block;margin:0 auto; } + .watch_video_holder { margin:0 0 20px 0; + .watch_demo { float:none; display:inline-block; margin:0 auto; } + } + .take_tour_holder { margin:0; } + .take_tour { .proxima_b(14px); color:@orange; text-decoration:none; text-transform:uppercase; letter-spacing: 2px; } + .take_tour:hover { color:#ff8720; } + */ + +} +#download_panel h2 { + width: 100%; + text-align: center; + color: #3F3E3E; + clear: both; + float: left; + margin-bottom: 44px; +} +#download_panel .download_holder { + display: inline-block; + float: none; + margin: 0 auto; + width: 100%; + text-align: center; +} +#download_panel .download { + clear: none; + margin: 0; + float: none; + display: inline-block; +} +#download_panel .watch_demo { + clear: none; + float: none; + margin: 0 0 0 20px; + display: inline-block; +} +#footer { + width: 100%; + clear: both; + float: left; + background: #FAF7F7; + padding: 70px 10% 40px; +} +#footer .content_frame { + max-width: 1400px; + width: 100%; + margin: 0 auto; +} +#footer ul { + float: left; + margin-bottom: 60px; + padding: 0; +} +#footer ul li { + float: left; + list-style: none; + margin: 0 60px 0 0; + padding: 0; +} +#footer ul li a { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + text-decoration: none; + color: #3F3E3E; +} +#footer ul li a:hover { + color: #ff9625; +} +#footer ul li.right { + float: right; +} +#footer ul li:last-child { + margin-right: 0; +} +#footer ul.share_nav { + float: right; +} +#footer ul.share_nav li { + margin: 0 0 0 60px; +} +#footer ul.share_nav li:first-child { + margin-left: 0; +} +#footer .footer_note { + clear: both; + text-align: center; +} +#footer .footer_note .note_partial { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #9F9C9C; + text-align: left; + display: inline; +} +#footer .footer_note a { + color: #3F3E3E; + text-decoration: none; +} +#footer .footer_note a:hover { + color: #ff9625; +} +#footer .copyright a { + text-decoration: none; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #9F9C9C; +} +/* + * Secondary Page Styles + * + */ +#secondary-container { + color: #3F3E3E; +} +#secondary-container .content_frame { + max-width: 1400px; + margin: 0 auto; + text-align: left; + padding: 0 0 100px 0; +} +#secondary-container .copy_holder { + float: left; + max-width: 880px; + width: 100%; + padding: 0 10% 50px 10%; +} +#secondary-container h2 { + color: #3F3E3E; + margin-top: 50px; + width: 100%; +} +#secondary-container .last-modified { + color: #9F9C9C; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + width: 100%; +} +#secondary-container .separator { + width: 100%; + height: 2px; + margin: 15px 0 45px; + border-bottom: 1px solid #A09C9C; +} +#secondary-container p { + color: #3F3E3E; + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + width: 100%; + line-height: 21px; +} +#secondary-container a { + color: #3F3E3E; +} +#secondary-container h3 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 22px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + width: 100%; + margin: 40px 0 20px; + line-height: 26px; +} +#secondary-container h4 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + width: 100%; + margin: 40px 0 10px; +} +#secondary-container ul { + width: 100%; +} +#secondary-container ul li { + color: #3F3E3E; + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + width: 100%; + line-height: 21px; + margin-bottom: 5px; +} +body.faq .faq-toc { + padding: 0; + margin: 0; +} +body.faq .faq-toc li { + margin: 0; + padding: 0; + list-style: none; +} +body.faq .faq-toc li a { + text-decoration: none; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; +} +body.faq .faq-list { + padding: 0; + margin: 0; +} +body.faq .faq-item { + list-style: none; + padding: 0; + margin: 0; +} +@media screen and (max-width: 1100px) { + h2 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 26px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + font-weight: 500; + line-height: 30px; + letter-spacing: 0; + } + #footer { + padding: 30px 10% 40px; + } + #footer ul { + float: left; + margin: 0; + width: 100%; + text-align: center; + } + #footer ul li { + margin: 0 30px 0 0; + float: none; + display: inline; + } + #footer ul.share_nav { + clear: both; + width: 100%; + margin: 20px auto 30px; + text-align: center; + } + #footer ul.share_nav li { + margin: 0 0 0 30px; + display: inline; + float: none; + } + #footer .footer_note .crafted { + float: left; + width: 100%; + text-align: center; + display: block; + margin-bottom: 5px; + } +} +@media screen and (max-width: 1000px) { + #top_nav ul li { + margin: 0 0 0 35px; + } + #top_nav ul li:last-child { + margin-left: 35px; + } +} +@media screen and (max-width: 900px) { + #top_nav h1 { + width: 213px; + height: 35px; + float: left; + margin-left: 5%; + } + #top_nav ul { + margin-right: 5%; + } + #top_nav ul li { + margin: 0 0 0 25px; + } + #top_nav ul li:last-child { + margin-left: 25px; + } + #secondary-container .content_frame { + padding: 0 0 50px; + } + #secondary-container .copy_holder { + padding-left: 5%; + } +} +@media screen and (max-width: 800px) { + #top_nav ul li { + margin: 0 0 0 15px; + } + #top_nav ul li a { + letter-spacing: 1px; + } + /* + #top_nav { + .nav_holder { clear:both; float:left; width:100%; margin:27px 0 0; height:0; overflow:hidden; } + ul { clear:both; float:left; margin:0; padding:35px 0;width:100%; text-align:center;border-top:1px solid #CFCBCB; + li { float:none; display:inline; margin:0 0 0 65px; } + li.home { display:none; } + li:first-child { margin-left:0; } + li:last-child { margin-left:65px; } + } + .menu { display:inline; margin-right:5%; } + } + */ +} +@media screen and (max-width: 700px) { + #top_nav ul li { + margin: 0 0 0 5px; + } + #top_nav ul li a { + letter-spacing: 0px; + } +} +@media screen and (max-width: 600px) { + #top_nav { + min-height: 66px; + /* + .menu { margin-top:17px; } + .nav_holder { margin-top:17px; } + .watch_demo_link { margin-top:25px; } + */ + + } + #top_nav h1 { + width: 36px; + height: 36px; + margin-top: 15px; + } + #top_nav h1 a { + background: url('/static/marketing/images/logo_icon@2x.png') 0 0 no-repeat; + background-size: 34px 34px; + background-position: 0 0; + background-repeat: no-repeat; + } + #top_nav ul { + margin-top: 17px; + } + #download_panel { + padding: 25px 0 40px 0; + } + #download_panel h2 { + width: 70%; + float: none; + display: inline-block; + margin: 20px auto 44px; + max-width: 230px; + } + #download_panel .download_holder { + max-width: 400px; + } + #download_panel .watch_demo { + margin: 22px 0 0 0; + } + #footer { + text-align: center; + } + #footer .footer_note .copyright, + #footer .footer_note .terms { + float: left; + width: 100%; + text-align: center; + display: block; + margin-bottom: 5px; + } + #footer ul.footer_nav { + width: auto; + float: none; + display: inline-block; + margin: 0 auto; + } + #footer ul.footer_nav li { + margin-bottom: 20px; + float: left; + } + #footer ul.footer_nav .journal { + margin-right: 0; + } +} +/** + * Homepage Styling + * + */ +.homepage #download_panel .download_holder { + display: inline-block; + float: none; + margin: 0 auto; + width: 100%; + text-align: center; +} +.homepage #download_panel .download { + clear: none; + margin: 0; + float: none; + display: inline-block; +} +.homepage #download_panel .watch_demo { + border: none; + background: none; + clear: none; + float: none; + margin: 22px 0 0 0; + display: inline-block; + color: #3F3E3E; + padding: 0; +} +.homepage #download_panel .watch_demo .icon { + margin: 0 0 -2px 5px; +} +#home_hero { + height: 570px; + overflow: hidden; + background: #000; + text-align: center; + position: relative; +} +#home_hero .bkgd_img, +#home_hero .bkgd_img_off { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-position: center 0; + background-repeat: no-repeat; + z-index: 0; +} +#home_hero .bkgd_img_off { + display: none; +} +#home_hero .content_frame { + position: relative; + z-index: 2; + margin: 0 auto; + max-width: 1400px; + padding: 0 10%; + width: 100%; + display: inline-block; +} +#home_hero h2 { + color: #fff; + float: left; + position: relative; + z-index: 2; + text-align: left; + width: 400px; + margin: 230px 0 0 0; +} +#home_hero .download { + position: relative; + z-index: 2; + clear: both; + float: left; + margin: 44px 0 0 0; +} +#home_hero .bottom_gradient { + display: none; + position: absolute; + bottom: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 1; + background: url(data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/Pgo8c3ZnIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDEgMSIgcHJlc2VydmVBc3BlY3RSYXRpbz0ibm9uZSI+CiAgPGxpbmVhckdyYWRpZW50IGlkPSJncmFkLXVjZ2ctZ2VuZXJhdGVkIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgeDE9IjAlIiB5MT0iMCUiIHgyPSIwJSIgeTI9IjEwMCUiPgogICAgPHN0b3Agb2Zmc2V0PSIwJSIgc3RvcC1jb2xvcj0iIzAwMDAwMCIgc3RvcC1vcGFjaXR5PSIwIi8+CiAgICA8c3RvcCBvZmZzZXQ9Ijk5JSIgc3RvcC1jb2xvcj0iIzAwMDAwMCIgc3RvcC1vcGFjaXR5PSIwLjY0Ii8+CiAgICA8c3RvcCBvZmZzZXQ9IjEwMCUiIHN0b3AtY29sb3I9IiMwMDAwMDAiIHN0b3Atb3BhY2l0eT0iMC42NSIvPgogIDwvbGluZWFyR3JhZGllbnQ+CiAgPHJlY3QgeD0iMCIgeT0iMCIgd2lkdGg9IjEiIGhlaWdodD0iMSIgZmlsbD0idXJsKCNncmFkLXVjZ2ctZ2VuZXJhdGVkKSIgLz4KPC9zdmc+); + background: -moz-linear-gradient(top, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.64) 99%, rgba(0, 0, 0, 0.65) 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, rgba(0, 0, 0, 0)), color-stop(99%, rgba(0, 0, 0, 0.64)), color-stop(100%, rgba(0, 0, 0, 0.65))); + background: -webkit-linear-gradient(top, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.64) 99%, rgba(0, 0, 0, 0.65) 100%); + background: -o-linear-gradient(top, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.64) 99%, rgba(0, 0, 0, 0.65) 100%); + background: -ms-linear-gradient(top, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.64) 99%, rgba(0, 0, 0, 0.65) 100%); + background: linear-gradient(to bottom, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.64) 99%, rgba(0, 0, 0, 0.65) 100%); + filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#a6000000', GradientType=0); +} +#tour_panel { + padding: 130px 8% 130px; +} +#tour_panel .content_frame { + border: 15px solid #FAF7F7; + width: 100%; + height: auto; + max-width: 1000px; + margin: 0 auto; +} +#tour_panel .content_frame .phone { + position: relative; + float: right; + width: 382px; + height: 732px; + margin: -95px 0 -95px 40px; +} +#tour_points { + float: right; + text-align: left; + margin: 132px 0px 0 85px; + width: 330px; +} +#tour_points h2 { + margin: 0 0 22px 0px; + width: 100%; +} +#tour_points ul { + list-style: none; + margin: 0; + padding: 0; + width: 100%; + margin-left: 1px; +} +#tour_points ul li { + list-style: none; + margin: 0 0 22px 0; + padding: 0; + position: relative; +} +#tour_points ul li a { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #bcbcbc; + padding: 0; + line-height: 20px; + text-decoration: none; + position: relative; +} +#tour_points ul li .radio { + position: absolute; + top: 1px; + left: -40px; + background: url(/static/marketing/images/radio_sprite@2x.png) -43px -44px no-repeat; + background-size: 80px 86px; + width: 21px; + height: 21px; +} +#tour_points ul li:hover a { + color: #ff9625; +} +#tour_points ul li:hover .radio { + background-position: -43px -66px; +} +#tour_points ul li.current a { + color: #3F3E3E; +} +#tour_points ul li.current .radio { + background-position: -43px 0; +} +#tour_points ul li.current:hover a { + color: #ff9625; +} +#tour_points ul li.current:hover .radio { + background-position: -43px -22px; +} +#tour_points .radio_buttons_small { + display: none; + float: left; + width: auto; +} +#tour_points .radio_buttons_small li { + width: 11px; + height: 10px; + float: left; + margin-right: 10px; +} +#tour_points .radio_buttons_small li .radio { + position: relative; + top: 0; + left: 0; + float: left; + background: url(/static/marketing/images/radio_sprite@2x.png) -69px -49px no-repeat; + width: 11px; + height: 10px; + background-size: 80px 86px; +} +#tour_points .radio_buttons_small li:hover a { + color: #ff9625; +} +#tour_points .radio_buttons_small li:hover .radio { + background-position: -69px -71px; +} +#tour_points .radio_buttons_small li.current a { + color: #3F3E3E; +} +#tour_points .radio_buttons_small li.current .radio { + background-position: -69px -5px; +} +#tour_points .radio_buttons_small li.current:hover a { + color: #ff9625; +} +#tour_points .radio_buttons_small li.current:hover .radio { + background-position: -69px -27px; +} +#tour_points .radio_buttons_small li:last-child { + margin-right: 0; +} +#tour_video video { + float: left; +} +#tour_copy { + padding: 0; + height: 100%; + display: inline-block; + margin: 0 auto; +} +#tour_video_holder { + background-position: 0 0; + background-repeat: no-repeat; + width: 268px; + height: 476px; + position: absolute; + top: 128px; + left: 57px; + overflow: hidden; +} +#tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg'); + background-size: 268px 476px; + } +} +#tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg'); + background-size: 268px 476px; + } +} +#tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg'); +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg'); + background-size: 268px 476px; + } +} +#dial_panel { + background: #232223; + overflow: hidden; + height: 480px; +} +#dial_panel .bkgd_img { + background-image: url('/static/marketing/images/Home-02-iPhone-Dial.jpg'); + background-position: right top; + background-repeat: no-repeat; + height: 480px; + margin: 0 auto; + width: 100%; + min-width: 900px; + z-index: 0; +} +@media (-webkit-min-device-pixel-ratio: 1.5) { + #dial_panel .bkgd_img { + background-image: url('/static/marketing/images/Home-02-iPhone-Dial@2x.jpg'); + background-size: 1200px 836px; + } +} +#dial_panel .content_frame_abs { + position: absolute; + z-index: 1; + width: 100%; + top: 0; + left: 0; +} +#dial_panel .content_frame { + position: relative; + z-index: 1; + padding: 0 10%; + width: 100%; + margin: 0 auto; + max-width: 1400px; + display: inline-block; +} +#dial_panel .content_frame:hover { + cursor: pointer; +} +#dial_panel h2 { + color: #fff; + float: left; + position: relative; + z-index: 1; + width: 300px; + margin: 160px 0 22px 0; +} +#dial_panel p { + color: #fff; + clear: both; + float: left; + position: relative; + width: 375px; + padding: 0; + margin: 0; +} +#dial_panel .watch_demo { + margin-top: 22px; +} +#dial_panel .bkgd_video { + width: 100%; + position: absolute; + top: 0; + left: 0; +} +#dial_panel .close { + position: absolute; + top: 15px; + left: 10px; + width: 33px; + height: 33px; + z-index: 10; + display: none; +} +#dial_panel.expanded .content_frame:hover { + cursor: default; +} +#dial_panel.has_video { + background: #413e40; +} +#dial_panel.has_video .bkgd_img { + background: #413e40; +} +#dial_video_holder { + float: left; + display: none; +} +#dial_bkgd_video { + float: right; +} +#preserve_panel { + padding: 130px 8%; +} +#preserve_panel .content_panel { + position: absolute; + z-index: 1; + top: 0; + left: 0; + padding: 0 10%; +} +#preserve_panel .content_frame { + border: 15px solid #FAF7F7; + width: 100%; + height: 572px; + text-align: center; + max-width: 1000px; + margin: 0 auto; +} +#preserve_panel .content_frame h2 { + color: #ff9625; + float: left; + position: relative; + z-index: 1; + width: 300px; + margin: 120px 0 0 0; +} +#preserve_panel .content_frame p { + color: #3F3E3E; + clear: both; + float: left; + position: relative; + width: 300px; + margin: 22px 0 0 0; + padding: 0; +} +#preserve_panel .content_frame .download { + position: relative; + z-index: 1; + clear: both; + float: left; + margin: 22px 0 0 0; +} +#preserve_copy { + padding: 0 0 0 405px; + height: 100%; + background: url(/static/marketing/images/viewFinder_amber-small400.png) 0 100px no-repeat; + display: inline-block; + margin: 0 auto; +} +@media screen and (max-width: 1100px) { + #home_hero { + height: 352px; + } + #home_hero h2 { + width: 250px; + margin: 110px 0 0 0; + } + #home_hero .download { + margin: 22px 0 0 0; + } + #tour_panel { + padding: 30px 8% 10px; + } + #tour_panel .content_frame { + border: none; + width: 100%; + } + #tour_panel .content_frame .phone { + width: 282px; + height: 542px; + margin: 0; + } + #tour_points { + margin: 120px 30px 0 20px; + width: 240px; + } + #tour_points ul li .radio { + display: none; + } + #tour_points .radio_buttons_small { + display: inline; + } + #tour_points .radio_buttons_small li .radio { + display: inline; + } + #tour_video_holder { + width: 201px; + height: 357px; + top: 91px; + left: 40px; + } + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg'); + background-size: 201px 357px; + } + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg'); + background-size: 201px 357px; + } + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg'); + background-size: 201px 357px; + } + #dial_panel { + height: 370px; + } + #dial_panel .bkgd_img { + background: url('/static/marketing/images/Home-02-iPhone-Dial.jpg') right top no-repeat; + background-size: 535px 373px; + height: 370px; + } + #dial_panel h2 { + width: 330px; + } + #dial_panel p { + width: 330px; + } + #preserve_panel { + padding: 0 8% 115px; + overflow: hidden; + } + #preserve_panel .content_frame { + border: none; + height: auto; + } + #preserve_panel .content_frame h2 { + width: 235px; + margin: 115px 0 0 0; + } + #preserve_panel .content_frame p { + width: 235px; + } + #preserve_panel .content_frame .download { + margin: 22px 0 0 0; + } + #preserve_copy { + padding: 0 30px 0 360px; + background-position: 0 70px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_1 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_2 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1100px) and (-webkit-min-device-pixel-ratio: 1.5) { + #tour_video_holder.point_3 { + background-image: url('/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg'); + background-size: 201px 357px; + } +} +@media screen and (max-width: 1000px) { + +} +@media screen and (max-width: 900px) { + #tour_panel { + padding-left: 2%; + padding-right: 2%; + } +} +@media screen and (max-width: 800px) { + +} +@media screen and (max-width: 600px) { + #home_hero h2 { + width: 100%; + margin-top: 180px; + text-align: center; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 20px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + font-weight: 500; + line-height: 24px; + } + #home_hero .download { + margin: 22px auto 0; + float: none; + display: inline-block; + } + #home_hero .bottom_gradient { + display: block; + } + #tour_panel .content_frame { + text-align: center; + } + #tour_panel .content_frame .phone { + margin: 0 auto; + display: inline-block; + float: none; + } + #tour_points { + margin: 0; + width: 100%; + text-align: center; + } + #tour_points h2 { + width: 100%; + text-align: center; + } + #tour_points ul.points { + margin-bottom: 20px; + text-align: center; + float: left; + } + #tour_points ul.points li { + display: none; + margin: 0 auto; + text-align: center; + width: 70%; + margin-left: 15%; + float: left; + min-height: 40px; + } + #tour_points ul.points li.current { + display: inline; + } + #tour_points .radio_buttons_small { + display: inline-block; + float: none; + } + #tour_points .radio_buttons_small li.tour_point { + margin-bottom: 5px; + } + #dial_panel { + height: 300px; + } + #dial_panel .bkgd_img { + /*background:url(/static/marketing/images/dial_bkgd_small.png) right top no-repeat; */ + background: none; + height: auto; + min-width: 300px; + } + #dial_panel h2 { + margin-top: 66px; + width: 100%; + text-align: center; + } + #dial_panel p { + width: 100%; + text-align: center; + } + #dial_panel .watch_demo { + float: none; + display: inline-block; + } + #dial_bkgd_video { + display: none; + } + #preserve_panel { + padding: 0 0; + overflow: hidden; + } + #preserve_panel .content_frame { + border: none; + width: 100%; + margin: 0 auto; + } + #preserve_panel .content_frame h2 { + width: 84%; + text-align: center; + margin: 30px 0 0 8%; + } + #preserve_panel .content_frame p { + width: 84%; + text-align: center; + margin-left: auto; + margin-right: auto; + display: inline-block; + float: none; + } + #preserve_panel .content_frame .download { + margin: 22px auto 0; + float: none; + display: inline-block; + } + #preserve_copy { + padding: 0 0 300px 0; + background-position: center bottom; + margin-bottom: 20px; + text-align: center; + } +} +@media screen and (max-width: 480px) { + #dial_panel .content_frame { + padding: 0 5%; + } + #dial_panel .bkgd_img { + height: 300px; + } +} +/** + * Tour Page Styling + * + */ +.tour_panel h3 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 42px; + font-weight: 300; + -webkit-font-smoothing: antialiased; + margin: 0; + padding: 0; +} +.tour_panel h4 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 22px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + margin: 0; + padding: 0; + letter-spacing: 3px; +} +.tour_panel p { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #3F3E3E; +} +.tour_panel .content_frame { + max-width: 1400px; + margin: 0 auto; +} +.tour_panel .dot { + width: 8px; + height: 8px; + background: url(/static/marketing/images/tour/dot@2x.png) 0 0 no-repeat; + background-size: 8px 8px; +} +.tour_panel .divider { + height: 50px; + clear: both; + display: inline-block; + width: 115px; + border-bottom: 1px solid #CFCBCB; + position: relative; + display: none; +} +.tour_panel .divider .dot { + position: absolute; + bottom: -4px; + left: 49%; +} +#tour_intro_panel { + padding: 20px 0 300px; + text-align: center; + overflow: hidden; + /* + .iphone_holder.disabled { + .iphone:hover { cursor:default; } + .screenshot:hover { cursor:default; } + } +*/ + +} +#tour_intro_panel .content_frame { + position: relative; +} +#tour_intro_panel .intro_copy { + max-width: 520px; + width: 36%; + display: inline-block; + text-align: center; + z-index: 0; + position: absolute; +} +#tour_intro_panel .intro_copy h2 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 60px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #3F3E3E; + line-height: 64px; + text-align: center; + padding: 0; + margin-top: 135px; + max-width: 450px; + display: inline-block; +} +#tour_intro_panel .intro_copy p { + text-align: center; + max-width: 360px; + display: inline-block; +} +#tour_intro_panel .intro_copy .download { + display: inline-block; + margin-top: 10px; +} +#tour_intro_panel .intro_copy .continue_tour { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-transform: uppercase; + color: #CFCBCB; + display: inline-block; + clear: both; + width: 100%; + margin: 0 auto; + text-decoration: none; + margin-top: 55px; +} +#tour_intro_panel .intro_copy .continue_tour img { + margin-top: 10px; +} +#tour_intro_panel .intro_copy .copy_mask { + width: 100%; + height: 100%; + position: absolute; + top: 0; + left: 0; + background: #fff; + width: 0; +} +#tour_intro_panel .iphone_holder { + width: 382px; + height: 732px; + position: relative; + z-index: 2; +} +#tour_intro_panel .iphone_holder .iphone { + position: absolute; + top: 0; + left: 0; +} +#tour_intro_panel .iphone_holder .iphone:hover { + cursor: pointer; +} +#tour_intro_panel .iphone_holder .white_mask { + height: 100%; + position: absolute; + top: -40px; + width: 800px; + background: #fff; +} +#tour_intro_panel .iphone_holder .caption_holder { + width: 320px; + height: 235px; + border-left: 1px solid #CFCBCB; + position: absolute; + bottom: -195px; +} +#tour_intro_panel .iphone_holder .caption_holder .copy { + position: absolute; + bottom: -5px; + text-align: left; +} +#tour_intro_panel .iphone_holder .caption_holder .copy h4 { + color: #3F3E3E; +} +#tour_intro_panel .iphone_holder .caption_holder .copy h4:hover { + cursor: pointer; +} +#tour_intro_panel .iphone_holder .caption_holder .copy p { + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #3F3E3E; +} +#tour_intro_panel .iphone_holder .caption_holder .copy a { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #9f9c9c; + text-decoration: none; + text-transform: uppercase; + padding: 0; + letter-spacing: 2px; +} +#tour_intro_panel .iphone_holder .caption_holder .dot { + position: absolute; + bottom: 0; +} +#tour_intro_panel .iphone_holder .screenshot { + width: 268px; + height: 476px; + position: absolute; + top: 128px; + left: 57px; +} +#tour_intro_panel .iphone_holder .screenshot:hover { + cursor: pointer; +} +#tour_intro_panel .tour_iphone_library { + float: left; + margin-left: -75px; +} +#tour_intro_panel .tour_iphone_library .white_mask { + right: 100px; +} +#tour_intro_panel .tour_iphone_library .caption_holder { + left: 191px; +} +#tour_intro_panel .tour_iphone_library .caption_holder .copy { + left: 18px; +} +#tour_intro_panel .tour_iphone_library .caption_holder .copy h4 img { + margin: 0 6px -3px 0; +} +#tour_intro_panel .tour_iphone_library .caption_holder .dot { + left: -4px; +} +#tour_intro_panel .tour_iphone_inbox { + float: right; + margin-right: -75px; +} +#tour_intro_panel .tour_iphone_inbox .white_mask { + left: 100px; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder { + right: 191px; + border-left: none; + border-right: 1px solid #cfcbcb; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .copy { + right: 18px; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .copy h4 { + text-align: right; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .copy h4 img { + margin: 0 6px -8px 0; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .copy p { + text-align: right; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .copy a { + text-align: right; + float: right; +} +#tour_intro_panel .tour_iphone_inbox .caption_holder .dot { + right: -4px; +} +#tour_intro_panel .detail_holder { + position: absolute; + width: 30%; + max-width: 385px; + top: 100px; + text-align: left; + z-index: 1; + opacity: 0; + display: none; +} +#tour_intro_panel .detail_holder .detail_mask { + width: 100%; + height: 100%; + position: absolute; + top: 0; + background: #fff; +} +#tour_intro_panel .detail_holder h3 { + margin-top: 15px; +} +#tour_intro_panel .detail_holder h4 img { + margin: 0 5px -3px 0; +} +#tour_intro_panel .detail_holder ul { + margin: 0 0 0 20px; + padding: 0; +} +#tour_intro_panel .detail_holder ul li { + list-style: none; + font-family: "freight-text-pro", "Times New Roman", Times, serif; + font-size: 16px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + color: #3F3E3E; + padding: 0 0 0 0; + margin: 0 0 20px 0; + position: relative; +} +#tour_intro_panel .detail_holder ul li .arrow { + position: absolute; + top: 5px; + left: -17px; +} +#tour_intro_panel .detail_holder .close { + color: #C73926; + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + text-decoration: none; + margin-top: 15px; + float: left; + text-transform: uppercase; + letter-spacing: 2px; +} +#tour_intro_panel .detail_inbox { + left: 50%; +} +#tour_intro_panel .detail_inbox .detail_mask { + left: 0; +} +#tour_intro_panel .detail_inbox h4 img { + margin: 0 5px -7px 0; +} +#tour_intro_panel .detail_library { + right: 50%; +} +#tour_intro_panel .detail_library .detail_mask { + right: 0; +} +#sharing_panel { + background: #2F2E2E; + padding: 60px 0 40px; +} +#sharing_panel h3 { + color: #fff; + max-width: 450px; + margin: 0 auto; + display: block; +} +#sharing_panel p { + color: #fff; + max-width: 450px; + display: block; + margin: 15px auto 50px; +} +#sharing_panel ul { + display: inline-block; + margin: 0 auto; + max-width: 1175px; + width: 100%; + padding: 0; +} +#sharing_panel ul li { + list-style: none; + float: left; +} +#sharing_panel ul li.arrow { + margin-top: 120px; +} +#sharing_panel ul li.arrow .down { + display: none; +} +#sharing_panel ul li.step { + padding: 0 35px; + min-width: 290px; + width: 30%; +} +#sharing_panel ul li.step .screenshot { + margin-bottom: 40px; +} +#sharing_panel ul li.step .copy { + width: 210px; + display: inline-block; +} +#sharing_panel ul li.step h4 { + color: #fff; + text-align: left; + float: left; + width: 100%; +} +#sharing_panel ul li.step h4 img { + margin: 0 6px -3px 0; +} +#sharing_panel ul li.step p { + clear: both; + float: left; + width: 100%; + max-width: 200px; +} +#sharing_panel ul li.start h4 img { + margin-bottom: -6px; +} +#sharing_panel ul li.share .screenshot { + margin-top: 26px; + margin-bottom: 14px; +} +#sharing_panel ul li.share h4 img { + margin-bottom: -7px; +} +#sharing_panel ul li.save .screenshot { + margin-left: -27px; +} +#organized_panel { + padding: 100px 0 100px 0; + border-bottom: 1px solid #CFCBCB; + background-color: #FAF7F7; +} +#organized_panel .content_frame { + max-width: 790px; + padding: 0 10px; + width: 100%; +} +#organized_panel .stream_holder { + height: 630px; + width: 328px; + float: left; + overflow: hidden; +} +#organized_panel .copy { + float: right; + width: 320px; + margin-top: 0px; +} +#organized_panel .copy h3 { + text-align: left; + margin: 30px 0 40px 0; +} +#organized_panel .copy ul { + padding: 0; +} +#organized_panel .copy ul li { + list-style: none; + float: left; + text-align: left; + padding: 0; + color: #3F3E3E; + margin-bottom: 25px; +} +#organized_panel .copy ul li h4 img { + margin: 0 6px -2px 0; +} +#organized_panel .copy ul li p { + margin: 12px 0 0; + padding: 0; + max-width: 300px; +} +#organized_panel .copy ul li.searchable h4 img { + margin-bottom: -4px; +} +#organized_panel .copy ul li.private h4 img { + margin-bottom: -3px; +} + +#organized_panel_2 { + padding: 100px 0 100px 0; + border-bottom: 1px solid #CFCBCB; +} +#organized_panel_2 .content_frame { + max-width: 790px; + padding: 0 10px; + width: 100%; +} +#organized_panel_2 .stream_holder { + height: 630px; + width: 328px; + float: right; + overflow: hidden; +} +#organized_panel_2 .copy { + float: left; + width: 360px; + margin-top: 0px; +} +#organized_panel_2 .copy h3 { + text-align: left; + margin: 60px 0 40px 0; +} +#organized_panel_2 .copy ul { + padding: 0; +} +#organized_panel_2 .copy ul li { + list-style: none; + float: left; + text-align: left; + padding: 0; + color: #3F3E3E; + margin-bottom: 25px; +} +#organized_panel_2 .copy ul li h4 img { + margin: 0 6px -2px 0; +} +#organized_panel_2 .copy ul li p { + margin: 12px 0 0; + padding: 0; + max-width: 220px; +} +#organized_panel_2 .copy ul li.searchable h4 img { + margin-bottom: -4px; +} +#organized_panel_2 .copy ul li.private h4 img { + margin-bottom: -3px; +} + +#conversation_panel { + padding: 85px 0 110px; + overflow: hidden; + border-bottom: 1px solid #CFCBCB; +} +#conversation_panel .content_frame { + max-width: 1400px; + width: 100%; + margin: 0 auto; + position: relative; +} +#conversation_panel ul.phone_list { + list-style: none; + padding: 0; + float: left; + width: 100%; + height: 700px; + position: relative; + margin: 0; +} +#conversation_panel ul.phone_list li { + float: left; + margin: 0; + padding: 0; + position: absolute; + top: 0; +} +#conversation_panel ul.phone_list li .bkgd_img { + width: 382px; + height: 732px; +} +#conversation_panel ul.phone_list li .screenshot { + position: absolute; + top: 126px; + left: 56px; + width: 270px; + height: 470px; + color: #fff; +} +#conversation_panel ul.phone_list li.iphone_holder { + width: 382px; + height: 732px; +} +#conversation_panel ul.phone_list li.iphone_holder:hover { + cursor: pointer; +} +#conversation_panel ul.phone_list li.phone0 { + left: -15%; +} +#conversation_panel ul.phone_list li.phone1 { + left: 55%; +} +#conversation_panel ul.phone_list li.phone2 { + left: 85%; +} +#conversation_panel ul.phone_list li.phone3 { + left: 115%; +} +#conversation_panel ul.phone_list li.phone4 { + left: 145%; +} +#conversation_panel .caption { + text-align: left; + width: 30%; + height: 600px; + max-width: 385px; + position: absolute; + left: 18%; + top: 125px; +} +#conversation_panel .caption h3 { + clear: both; + float: left; + margin: 0 0 10px 0; + padding: 0; +} +#conversation_panel .caption p { + clear: both; + float: left; + margin: 11px 0; +} +#conversation_panel .caption .examples { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + color: #9F9C9C; + text-transform: uppercase; + text-decoration: none; + margin: 24px 0 0 0; + clear: both; + float: left; +} +#conversation_panel .caption .examples .arrow { + margin: 0 0 1px 3px; +} +#conversation_panel .caption .convo_nav { + width: 100%; + height: 100px; + position: absolute; + bottom: 0; + left: 0; + padding: 0; + text-align: center; +} +#conversation_panel .caption .convo_nav .top_rule { + border-top: 1px solid #CFCBCB; + width: 100%; + position: relative; + float: left; + height: 40px; +} +#conversation_panel .caption .convo_nav .top_rule .dot { + position: absolute; + top: -4px; + left: 49%; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list { + list-style: none; + padding: 0; + display: inline-block; + margin: 0; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li { + background: url(/static/marketing/images/radio_sprite@2x.png) -43px -44px no-repeat; + background-size: 80px 86px; + width: 21px; + height: 21px; + float: left; + margin: 0 7px 0 8px; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li a { + width: 100%; + height: 100%; + float: left; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li:hover { + background-position: -43px -66px; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.current { + background-position: -43px 0; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.current:hover { + background-position: -43px -22px; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.left_arrow, +#conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow { + background: url(/static/marketing/images/radio_sprite@2x.png) 0 0 no-repeat; + background-size: 80px 86px; + width: 16px; + height: 18px; + margin-top: 1px; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow { + background-position: -24px 0; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.left_arrow:hover { + background-position: 0 -22px; +} +#conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow:hover { + background-position: -24px -22px; +} +#conversation_panel .caption .copy { + position: absolute; + top: 0; + left: 0; + display: none; +} +#conversation_panel .caption .copy .caption_mask { + position: absolute; + top: 0; + left: 0; + background: #fff; + width: 100%; + height: 100%; + display: none; +} +#other_panel { + padding: 60px 0 20px; +} +#other_panel .content_frame { + max-width: 1100px; + margin: 0 auto; +} +#other_panel h3 { + margin-bottom: 60px; +} +#other_panel ul { + list-style: none; +} +#other_panel ul li { + float: left; + max-width: 515px; + width: 48%; + padding: 0 20px 70px; +} +#other_panel ul li .other_icon { + float: left; + width: 88px; + height: 88px; + margin: 0; +} +#other_panel ul li .copy { + float: left; + text-align: left; + width: 72%; + max-width: 375px; + float: right; + padding-left: 10px; +} +#other_panel ul li .copy p { + margin-bottom: 0; +} +#tour_video_overlay { + background: #000; + width: 100%; + height: 100%; + position: fixed; + z-index: 5; + display: none; +} +#tour_video_overlay #dial_video_holder { + display: inline; +} +#tour_video_overlay .close { + position: absolute; + top: 15px; + left: 10px; + width: 33px; + height: 33px; + z-index: 10; +} +@media screen and (max-width: 1200px) { + #css_vp_wid { + width: 1200px; + } + #tour_intro_panel .intro_copy { + max-width: 450px; + } + #tour_intro_panel .intro_copy h2 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 40px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + line-height: 54px; + margin-top: 225px; + width: 100%; + } + #tour_intro_panel .detail_inbox { + left: 52%; + } + #tour_intro_panel .detail_library { + right: 52%; + } + #conversation_panel .caption { + left: 21%; + } +} +@media screen and (max-width: 1100px) { + #css_vp_wid { + width: 1100px; + } + #tour_intro_panel .intro_copy { + max-width: 430px; + } + #tour_intro_panel .intro_copy h2 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 32px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + line-height: 40px; + } + #tour_intro_panel .detail_inbox { + left: 55%; + } + #tour_intro_panel .detail_library { + right: 55%; + } + #conversation_panel .caption { + left: 23%; + } +} +@media screen and (max-width: 1000px) { + #css_vp_wid { + width: 1000px; + } + .tour_panel h3 { + font-family: "proxima-nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 26px; + font-weight: 400; + -webkit-font-smoothing: antialiased; + line-height: 30px; + } + #tour_intro_panel { + padding: 70px 0 0; + height: 1000px; + } + #tour_intro_panel .content_frame { + max-width: 600px; + } + #tour_intro_panel .intro_copy { + max-width: 100%; + width: 100%; + padding: 0 40px; + position: relative; + top: auto; + left: auto; + } + #tour_intro_panel .intro_copy h2 { + max-width: 100%; + margin: 0; + padding: 0; + max-width: 400px; + } + #tour_intro_panel .intro_copy p { + margin: 10px 0 20px; + } + #tour_intro_panel .intro_copy .continue_tour { + display: none; + } + #tour_intro_panel .iphone_holder { + margin: 250px 0 0; + } + #tour_intro_panel .iphone_holder .caption_holder { + width: 225px; + height: auto; + border: none; + position: absolute; + bottom: auto; + top: -10px; + } + #tour_intro_panel .iphone_holder .caption_holder .copy { + position: absolute; + bottom: 0; + /*min-height:200px;*/ + } + #tour_intro_panel .iphone_holder .caption_holder .dot { + display: none; + } + #tour_intro_panel .tour_iphone_library { + margin-left: -100px; + } + #tour_intro_panel .tour_iphone_library .caption_holder { + left: 0; + } + #tour_intro_panel .tour_iphone_library .caption_holder .copy { + left: 38px; + } + #tour_intro_panel .tour_iphone_inbox { + margin-right: -100px; + } + #tour_intro_panel .tour_iphone_inbox .caption_holder { + right: 0; + } + #tour_intro_panel .tour_iphone_inbox .caption_holder .copy { + right: 38px; + } + #tour_intro_panel .detail_holder { + width: 45%; + max-width: 385px; + top: 300px; + opacity: 0; + } + #tour_intro_panel .detail_holder .detail_mask { + display: none; + } + #tour_intro_panel .detail_inbox { + right: -60px; + left: auto; + } + #tour_intro_panel .detail_library { + left: -60px; + right: auto; + } + #sharing_panel { + padding-bottom: 70px; + } + #sharing_panel p { + margin: 15px auto 0; + } + #sharing_panel ul { + max-width: 100%; + margin-top: 70px; + text-align: center; + } + #sharing_panel ul li { + clear: both; + } + #sharing_panel ul li.arrow { + width: 100%; + margin: 35px auto; + } + #sharing_panel ul li.arrow .down { + display: inline-block; + } + #sharing_panel ul li.arrow .right { + display: none; + } + #sharing_panel ul li.arrow2 { + margin: 15px auto; + } + #sharing_panel ul li.step { + padding: 0 35px; + min-width: 520px; + width: auto; + display: inline-block; + margin: 0 auto; + float: none; + } + #sharing_panel ul li.step .screenshot { + margin: 0; + float: left; + } + #sharing_panel ul li.step .copy { + margin: 35px 0 0 50px; + float: left; + } + #sharing_panel ul li.share .screenshot { + margin-top: 0px; + margin-bottom: 0; + } + #sharing_panel ul li.share .copy { + margin-top: 9px; + } + #sharing_panel ul li.save { + padding: 0; + } + #sharing_panel ul li.save .screenshot { + margin-left: -6px; + } + #organized_panel { + padding: 46px 0 46px 0; + overflow: hidden; + } + #organized_panel .content_frame { + height: 570px; + max-width: 600px; + padding: 0; + width: 100%; + } + #organized_panel .stream_holder { + margin-left: -48px; + } + #organized_panel .copy { + padding-right: 20px; + width: 270px; + margin-right: 5px; + } + #organized_panel .copy h3 { + margin: 20px 0 24px 0; + } + + #organized_panel_2 { + padding: 46px 0 46px 0; + overflow: hidden; + } + #organized_panel_2 .content_frame { + height: 570px; + max-width: 630px; + padding: 0; + width: 100%; + } + #organized_panel_2 .stream_holder { + margin-left: -48px; + } + #organized_panel_2 .copy { + padding-right: 60px; + width: 320px; + margin-right: 0px; + } + #organized_panel_2 .copy h3 { + margin: 80px 0 24px 0; + } + + #conversation_panel { + padding: 40px 0 50px; + } + #conversation_panel .content_frame { + max-width: 600px; + } + #conversation_panel ul.phone_list { + width: 287px; + height: 549px; + } + #conversation_panel ul.phone_list li .bkgd_img { + width: 287px; + height: 550px; + } + #conversation_panel ul.phone_list li .screenshot { + width: 201px; + height: 476px; + top: 97px; + left: 43px; + } + #conversation_panel ul.phone_list li.iphone_holder { + width: 287px; + height: 549px; + left: 0; + } + #conversation_panel ul.phone_list li.phone1 { + display: none; + } + #conversation_panel ul.phone_list li.phone2 { + display: none; + } + #conversation_panel ul.phone_list li.phone3 { + display: none; + } + #conversation_panel ul.phone_list li.phone4 { + display: none; + } + #conversation_panel .caption { + width: 230px; + height: 550px; + max-width: 100%; + position: relative; + left: auto; + top: auto; + float: left; + margin: 35px 0 0 37px; + } + #conversation_panel .caption .convo_nav { + width: 287px; + height: 15px; + position: absolute; + bottom: 0; + left: -302px; + padding: 0; + text-align: center; + } + #conversation_panel .caption .convo_nav .top_rule { + display: none; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li { + background: url(/static/marketing/images/radio_sprite@2x.png) -69px -49px no-repeat; + background-size: 80px 86px; + width: 11px; + height: 10px; + margin: 0 5px 0 5px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li:hover { + background-position: -69px -71px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.current { + background-position: -69px -5px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.current:hover { + background-position: -69px -27px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.left_arrow, + #conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow { + width: 8px; + height: 9px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.left_arrow { + background-position: -5px -50px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow { + background-position: -27px -50px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.left_arrow:hover { + background-position: -5px -72px; + } + #conversation_panel .caption .convo_nav ul.convo_nav_list li.right_arrow:hover { + background-position: -27px -72px; + } + #other_panel { + padding: 60px 40px 20px; + } + #other_panel .content_frame { + max-width: 100%; + width: 100%; + } + #other_panel h3 { + margin-bottom: 30px; + } + #other_panel ul { + max-width: 520px; + display: inline-block; + } + #other_panel ul li { + clear: both; + max-width: 100%; + width: 100%; + padding: 0; + margin: 0 0 10px 0; + } + #other_panel ul li .other_icon { + margin: 0 0 40px 0; + } + #other_panel ul li .copy { + margin-top: 5px; + max-width: 100%; + padding: 0px; + } +} +@media screen and (max-width: 700px) { + #css_vp_wid { + width: 700px; + } + #tour_intro_panel { + padding: 70px 0 0; + height: 1000px; + } + #tour_intro_panel .detail_holder { + width: 45%; + max-width: 275px; + } +} +@media screen and (max-width: 600px) { + #css_vp_wid { + width: 600px; + } + #tour_intro_panel { + padding: 25px 0 0; + height: auto; + } + #tour_intro_panel .content_frame { + max-width: 600px; + } + #tour_intro_panel .intro_copy { + max-width: 100%; + width: 100%; + padding: 0 40px; + } + #tour_intro_panel .intro_copy h2 { + max-width: 260px; + margin: 0; + padding: 0; + } + #tour_intro_panel .intro_copy p { + margin: 10px 0 20px; + } + #tour_intro_panel .intro_copy .continue_tour { + display: none; + } + #tour_intro_panel .divider { + display: inline-block; + } + #tour_intro_panel .iphone_holder { + margin: 50px 0 0; + height: 430px; + overflow: hidden; + width: 100%; + padding-top: 180px; + } + #tour_intro_panel .iphone_holder .caption_holder { + max-width: 600px; + width: 100%; + padding: 0 40px; + text-align: center; + top: 0; + } + #tour_intro_panel .iphone_holder .caption_holder .copy { + text-align: center; + padding: 0; + width: 100%; + left: auto; + right: auto; + position: relative; + float: none; + display: inline-block; + max-width: 240px; + } + #tour_intro_panel .iphone_holder .caption_holder .copy h4 { + text-align: center; + } + #tour_intro_panel .iphone_holder .caption_holder .copy p { + text-align: center; + max-width: 210px; + display: inline-block; + } + #tour_intro_panel .iphone_holder .caption_holder .copy a { + float: none; + text-align: center; + } + #tour_intro_panel .iphone_holder .iphone { + position: relative; + float: none; + top: auto; + left: auto; + display: inline-block; + width: 287px; + height: 549px; + } + #tour_intro_panel .iphone_holder .screenshot { + width: 201px; + height: 476px; + position: absolute; + top: 97px; + left: 43px; + } + #tour_intro_panel .tour_iphone_library { + border-bottom: 1px solid #CFCBCB; + } + #tour_intro_panel .detail_holder { + width: 100%; + max-width: 250px; + top: 370px; + right: auto; + opacity: 0; + text-align: center; + } + #tour_intro_panel .detail_holder h3 { + text-align: center; + } + #tour_intro_panel .detail_holder h4 { + text-align: center; + } + #tour_intro_panel .detail_holder p { + text-align: justify; + } + #tour_intro_panel .detail_holder .close { + display: inline-block; + float: none; + } + #tour_intro_panel .detail_inbox { + top: 870px; + } + #sharing_panel { + padding-top: 25px; + } + #sharing_panel p { + margin: 20px auto 0; + padding: 0 40px; + text-align: center; + max-width: 330px; + } + #sharing_panel ul { + margin-top: 35px; + } + #sharing_panel ul li.arrow2 { + margin: 35px auto 15px; + } + #sharing_panel ul li.step { + min-width: 300px; + } + #sharing_panel ul li.step .screenshot { + float: none; + } + #sharing_panel ul li.step .copy { + clear: both; + text-align: center; + margin: 35px 0 0; + width: 100%; + } + #sharing_panel ul li.step .copy h4 { + float: none; + text-align: center; + } + #sharing_panel ul li.step .copy p { + max-width: 280px; + float: none; + display: inline-block; + } + #sharing_panel ul li.save .screenshot { + margin-left: -27px; + } + #organized_panel { + padding: 30px 0 230px 0; + } + #organized_panel .content_frame { + height: auto; + padding: 0; + width: 100%; + } + #organized_panel .stream_holder { + margin: 0; + position: absolute; + bottom: 0; + left: 0; + width: 100%; + height: 225px; + text-align: center; + } + #organized_panel .stream_holder .img_holder { + width: 246px; + display: inline-block; + } + #organized_panel .copy { + float: none; + width: 100%; + margin: 0; + padding: 0 30px; + text-align: center; + } + #organized_panel .copy h3 { + margin: 0px 0 24px 0; + text-align: center; + } + #organized_panel .copy ul { + max-width: 300px; + display: inline-block; + } + #organized_panel .copy ul li { + text-align: center; + float: none; + } + #organized_panel .copy ul li h4 { + text-align: center; + } + #organized_panel .copy ul li p { + text-align: center; + } + #organized_panel .copy ul li.private { + margin-bottom: 0; + } + + + #organized_panel_2 { + padding: 30px 0 230px 0; + } + #organized_panel_2 .content_frame { + height: auto; + padding: 0; + width: 100%; + } + #organized_panel_2 .stream_holder { + margin: 0; + position: absolute; + bottom: 0; + left: 0; + width: 100%; + height: 225px; + text-align: center; + } + #organized_panel_2 .stream_holder .img_holder { + width: 246px; + display: inline-block; + } + #organized_panel_2 .copy { + float: none; + width: 100%; + margin: 0; + padding: 16px 30px 30px; + text-align: center; + } + #organized_panel_2 .copy h3 { + margin: 0px 0 16px 0; + text-align: center; + } + + #organized_panel_2 .copy p { + display: inline-block; + max-width: 300px; + } + + #organized_panel_2 .copy ul { + max-width: 300px; + display: inline-block; + } + #organized_panel_2 .copy ul li { + text-align: center; + float: none; + } + #organized_panel_2 .copy ul li h4 { + text-align: center; + } + #organized_panel_2 .copy ul li p { + text-align: center; + } + #organized_panel_2 .copy ul li.private { + margin-bottom: 0; + } + + + #conversation_panel { + padding: 50px 0 0; + overflow: visible; + border: none; + border-bottom: 1px solid #CFCBCB; + z-index: 10; + } + #conversation_panel .content_frame { + padding: 0 40px 325px; + } + #conversation_panel .phone_list_holder { + width: 100%; + overflow: hidden; + position: absolute; + bottom: 0px; + left: 0px; + height: 310px; + } + #conversation_panel ul.phone_list { + position: absolute; + top: 0px; + left: 0px; + width: 1000px; + height: 310px; + float: none; + } + #conversation_panel ul.phone_list li .bkgd_img { + width: 191px; + height: 366px; + } + #conversation_panel ul.phone_list li .screenshot { + width: 135px; + height: 240px; + top: 63px; + left: 28px; + } + #conversation_panel ul.phone_list li.iphone_holder { + width: 191px; + height: 366px; + top: 0; + left: auto; + } + #conversation_panel ul.phone_list li.phone1 { + display: inline; + } + #conversation_panel ul.phone_list li.phone2 { + display: inline; + } + #conversation_panel ul.phone_list li.phone3 { + display: inline; + } + #conversation_panel ul.phone_list li.phone4 { + display: inline; + } + #conversation_panel .caption { + width: 100%; + height: auto; + float: left; + margin: 0 0 0 0; + } + #conversation_panel .caption .convo_nav { + width: 100%; + bottom: -355px; + left: 0; + } + #other_panel { + padding: 40px 40px 20px; + } + #other_panel .divider { + display: inline-block; + } + #other_panel h3 { + margin-top: 25px; + } + #other_panel ul { + max-width: 300px; + margin: 0; + padding: 0; + } + #other_panel ul li { + margin: 0 0 35px 0; + } + #other_panel ul li .other_icon { + margin: 0 0 7px 0; + float: none; + display: inline-block; + } + #other_panel ul li .copy { + width: 100%; + text-align: center; + } + #other_panel ul li .copy p { + text-align: center; + max-width: 215px; + display: inline-block; + } +} diff --git a/marketing/resources/static/marketing/images/Convo--Dinner.jpg b/marketing/resources/static/marketing/images/Convo--Dinner.jpg new file mode 100644 index 0000000..068a21f Binary files /dev/null and b/marketing/resources/static/marketing/images/Convo--Dinner.jpg differ diff --git a/marketing/resources/static/marketing/images/Convo--Dinner@2x.jpg b/marketing/resources/static/marketing/images/Convo--Dinner@2x.jpg new file mode 100644 index 0000000..02e4586 Binary files /dev/null and b/marketing/resources/static/marketing/images/Convo--Dinner@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Frame.png b/marketing/resources/static/marketing/images/Home-01-iPhone-Frame.png new file mode 100644 index 0000000..0562bc0 Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Frame.png differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Frame@2x.png b/marketing/resources/static/marketing/images/Home-01-iPhone-Frame@2x.png new file mode 100644 index 0000000..09610b6 Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Frame@2x.png differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg new file mode 100644 index 0000000..7c5be95 Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg new file mode 100644 index 0000000..cc603cd Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-A@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg new file mode 100644 index 0000000..c0c909a Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg new file mode 100644 index 0000000..3fe2665 Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-B@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg new file mode 100644 index 0000000..ec4d74a Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg new file mode 100644 index 0000000..e14de78 Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-01-iPhone-Screenshot-C@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-02-iPhone-Dial.jpg b/marketing/resources/static/marketing/images/Home-02-iPhone-Dial.jpg new file mode 100644 index 0000000..5f7b67c Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-02-iPhone-Dial.jpg differ diff --git a/marketing/resources/static/marketing/images/Home-02-iPhone-Dial@2x.jpg b/marketing/resources/static/marketing/images/Home-02-iPhone-Dial@2x.jpg new file mode 100644 index 0000000..64a33ca Binary files /dev/null and b/marketing/resources/static/marketing/images/Home-02-iPhone-Dial@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/Inbox.jpg b/marketing/resources/static/marketing/images/Inbox.jpg new file mode 100644 index 0000000..ab6a5c5 Binary files /dev/null and b/marketing/resources/static/marketing/images/Inbox.jpg differ diff --git a/marketing/resources/static/marketing/images/Inbox@2x.JPG b/marketing/resources/static/marketing/images/Inbox@2x.JPG new file mode 100644 index 0000000..6c49cd1 Binary files /dev/null and b/marketing/resources/static/marketing/images/Inbox@2x.JPG differ diff --git a/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_1.png b/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_1.png new file mode 100644 index 0000000..37cba8b Binary files /dev/null and b/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_1.png differ diff --git a/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_2.png b/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_2.png new file mode 100644 index 0000000..6b26d3f Binary files /dev/null and b/marketing/resources/static/marketing/images/V1-1_Website_HeroImage_2.png differ diff --git a/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite.png b/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite.png new file mode 100644 index 0000000..c51d35c Binary files /dev/null and b/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite.png differ diff --git a/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite@2x.png b/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite@2x.png new file mode 100644 index 0000000..dc68605 Binary files /dev/null and b/marketing/resources/static/marketing/images/Viewfinder-Icons-Sprite@2x.png differ diff --git a/marketing/resources/static/marketing/images/Viewfinder-Logo-Beta@2x.png b/marketing/resources/static/marketing/images/Viewfinder-Logo-Beta@2x.png new file mode 100644 index 0000000..258e3fc Binary files /dev/null and b/marketing/resources/static/marketing/images/Viewfinder-Logo-Beta@2x.png differ diff --git a/marketing/resources/static/marketing/images/Viewfinder-Logo.png b/marketing/resources/static/marketing/images/Viewfinder-Logo.png new file mode 100644 index 0000000..d798fa9 Binary files /dev/null and b/marketing/resources/static/marketing/images/Viewfinder-Logo.png differ diff --git a/marketing/resources/static/marketing/images/Viewfinder-Logo@2x.png b/marketing/resources/static/marketing/images/Viewfinder-Logo@2x.png new file mode 100644 index 0000000..826e26c Binary files /dev/null and b/marketing/resources/static/marketing/images/Viewfinder-Logo@2x.png differ diff --git a/marketing/resources/static/marketing/images/close_icon.png b/marketing/resources/static/marketing/images/close_icon.png new file mode 100644 index 0000000..f5d5753 Binary files /dev/null and b/marketing/resources/static/marketing/images/close_icon.png differ diff --git a/marketing/resources/static/marketing/images/dial_bkgd.jpg b/marketing/resources/static/marketing/images/dial_bkgd.jpg new file mode 100644 index 0000000..7a8e1e2 Binary files /dev/null and b/marketing/resources/static/marketing/images/dial_bkgd.jpg differ diff --git a/marketing/resources/static/marketing/images/dial_bkgd_small.jpg b/marketing/resources/static/marketing/images/dial_bkgd_small.jpg new file mode 100644 index 0000000..857e909 Binary files /dev/null and b/marketing/resources/static/marketing/images/dial_bkgd_small.jpg differ diff --git a/marketing/resources/static/marketing/images/dial_bkgd_small.png b/marketing/resources/static/marketing/images/dial_bkgd_small.png new file mode 100644 index 0000000..857e909 Binary files /dev/null and b/marketing/resources/static/marketing/images/dial_bkgd_small.png differ diff --git a/marketing/resources/static/marketing/images/fpo.amber.jpg b/marketing/resources/static/marketing/images/fpo.amber.jpg new file mode 100644 index 0000000..3a661b6 Binary files /dev/null and b/marketing/resources/static/marketing/images/fpo.amber.jpg differ diff --git a/marketing/resources/static/marketing/images/fpo.home_splash.jpg b/marketing/resources/static/marketing/images/fpo.home_splash.jpg new file mode 100644 index 0000000..8629e15 Binary files /dev/null and b/marketing/resources/static/marketing/images/fpo.home_splash.jpg differ diff --git a/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share.png b/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share.png new file mode 100644 index 0000000..821cdf5 Binary files /dev/null and b/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share.png differ diff --git a/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share@2x.png b/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share@2x.png new file mode 100644 index 0000000..abcb7e9 Binary files /dev/null and b/marketing/resources/static/marketing/images/iPhone-Image-for-Better-Share@2x.png differ diff --git a/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation.png b/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation.png new file mode 100644 index 0000000..7403720 Binary files /dev/null and b/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation.png differ diff --git a/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation@2x.png b/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation@2x.png new file mode 100644 index 0000000..b283a52 Binary files /dev/null and b/marketing/resources/static/marketing/images/iPhone-Image-for-Control-Conversation@2x.png differ diff --git a/marketing/resources/static/marketing/images/icon_arrow_right@2x.png b/marketing/resources/static/marketing/images/icon_arrow_right@2x.png new file mode 100644 index 0000000..8fdcc9a Binary files /dev/null and b/marketing/resources/static/marketing/images/icon_arrow_right@2x.png differ diff --git a/marketing/resources/static/marketing/images/logo.png b/marketing/resources/static/marketing/images/logo.png new file mode 100644 index 0000000..dacd3cf Binary files /dev/null and b/marketing/resources/static/marketing/images/logo.png differ diff --git a/marketing/resources/static/marketing/images/logo_icon.png b/marketing/resources/static/marketing/images/logo_icon.png new file mode 100644 index 0000000..de0d9b1 Binary files /dev/null and b/marketing/resources/static/marketing/images/logo_icon.png differ diff --git a/marketing/resources/static/marketing/images/logo_icon@2x.png b/marketing/resources/static/marketing/images/logo_icon@2x.png new file mode 100644 index 0000000..6f5339f Binary files /dev/null and b/marketing/resources/static/marketing/images/logo_icon@2x.png differ diff --git a/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero1.jpg b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero1.jpg new file mode 100644 index 0000000..c713d4c Binary files /dev/null and b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero1.jpg differ diff --git a/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero2.jpg b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero2.jpg new file mode 100644 index 0000000..97fc800 Binary files /dev/null and b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero2.jpg differ diff --git a/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero3.jpg b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero3.jpg new file mode 100644 index 0000000..976658b Binary files /dev/null and b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero3.jpg differ diff --git a/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero4.jpg b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero4.jpg new file mode 100644 index 0000000..d9c4b7a Binary files /dev/null and b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero4.jpg differ diff --git a/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero5.jpg b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero5.jpg new file mode 100644 index 0000000..435d4a1 Binary files /dev/null and b/marketing/resources/static/marketing/images/pemberton_viewFinder.coHero5.jpg differ diff --git a/marketing/resources/static/marketing/images/radio_arrows@2x.png b/marketing/resources/static/marketing/images/radio_arrows@2x.png new file mode 100644 index 0000000..672df18 Binary files /dev/null and b/marketing/resources/static/marketing/images/radio_arrows@2x.png differ diff --git a/marketing/resources/static/marketing/images/radio_sprite@2x.png b/marketing/resources/static/marketing/images/radio_sprite@2x.png new file mode 100644 index 0000000..9ee9ce1 Binary files /dev/null and b/marketing/resources/static/marketing/images/radio_sprite@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A.jpg new file mode 100644 index 0000000..edb2ed1 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A@2x.jpg new file mode 100644 index 0000000..729cbc2 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-A@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B.jpg new file mode 100644 index 0000000..b61c3c0 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B@2x.jpg new file mode 100644 index 0000000..0d10be5 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-B@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C.jpg new file mode 100644 index 0000000..6cde446 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C@2x.jpg new file mode 100644 index 0000000..61b1d9b Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-C@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D.jpg new file mode 100644 index 0000000..afb6f69 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D@2x.jpg new file mode 100644 index 0000000..234ee49 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Example-D@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro.jpg new file mode 100644 index 0000000..1c405c2 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro@2x.jpg new file mode 100644 index 0000000..12f9b84 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Convo-Intro@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black.png b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black.png new file mode 100644 index 0000000..53184d5 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black.png differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black@2x.png b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black@2x.png new file mode 100644 index 0000000..18b99f3 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-Black@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White.png b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White.png new file mode 100644 index 0000000..6320d4d Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White.png differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White@2x.png b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White@2x.png new file mode 100644 index 0000000..4987f4c Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Frame-White@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox.jpg new file mode 100644 index 0000000..081535b Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox@2x.jpg new file mode 100644 index 0000000..3cf0898 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Inbox@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library.jpg new file mode 100644 index 0000000..6664ffc Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library@2x.jpg new file mode 100644 index 0000000..d55a645 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-01-iPhone-Library@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save.jpg new file mode 100644 index 0000000..7571f9d Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save@2x.jpg new file mode 100644 index 0000000..8c99d77 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Save@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share.jpg new file mode 100644 index 0000000..a2ce3e9 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share@2x.jpg new file mode 100644 index 0000000..42c1e7e Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Share@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start.jpg new file mode 100644 index 0000000..660ad26 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start@2x.jpg new file mode 100644 index 0000000..b47c08e Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-02-Convo-Pullouts-Start@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream.jpg b/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream.jpg new file mode 100644 index 0000000..ccac09d Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream@2x.jpg b/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream@2x.jpg new file mode 100644 index 0000000..64cda86 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/Tour-03-Photo-Stream@2x.jpg differ diff --git a/marketing/resources/static/marketing/images/tour/continue_arrow@2x.png b/marketing/resources/static/marketing/images/tour/continue_arrow@2x.png new file mode 100644 index 0000000..43b51ff Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/continue_arrow@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/dot@2x.png b/marketing/resources/static/marketing/images/tour/dot@2x.png new file mode 100644 index 0000000..4461049 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/dot@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_inbox@2x.png b/marketing/resources/static/marketing/images/tour/icon_inbox@2x.png new file mode 100644 index 0000000..3fbd95f Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_inbox@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_library@2x.png b/marketing/resources/static/marketing/images/tour/icon_library@2x.png new file mode 100644 index 0000000..42c02b2 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_library@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_organize@2x.png b/marketing/resources/static/marketing/images/tour/icon_organize@2x.png new file mode 100644 index 0000000..37d50bd Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_organize@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_private@2x.png b/marketing/resources/static/marketing/images/tour/icon_private@2x.png new file mode 100644 index 0000000..28b24a6 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_private@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_save_white@2x.png b/marketing/resources/static/marketing/images/tour/icon_save_white@2x.png new file mode 100644 index 0000000..7f01ec7 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_save_white@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_search@2x.png b/marketing/resources/static/marketing/images/tour/icon_search@2x.png new file mode 100644 index 0000000..0296faf Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_search@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_share_white@2x.png b/marketing/resources/static/marketing/images/tour/icon_share_white@2x.png new file mode 100644 index 0000000..745c3bd Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_share_white@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/icon_start_white@2x.png b/marketing/resources/static/marketing/images/tour/icon_start_white@2x.png new file mode 100644 index 0000000..20a0431 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/icon_start_white@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_backup@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_backup@2x.png new file mode 100644 index 0000000..7d626ea Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_backup@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_everyone@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_everyone@2x.png new file mode 100644 index 0000000..00ddff1 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_everyone@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_everywhere@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_everywhere@2x.png new file mode 100644 index 0000000..6211772 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_everywhere@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_mute@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_mute@2x.png new file mode 100644 index 0000000..ee2d528 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_mute@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_search@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_search@2x.png new file mode 100644 index 0000000..959a79d Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_search@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/other_icon_unshare@2x.png b/marketing/resources/static/marketing/images/tour/other_icon_unshare@2x.png new file mode 100644 index 0000000..f1e64d4 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/other_icon_unshare@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/tour_arrow@2x.png b/marketing/resources/static/marketing/images/tour/tour_arrow@2x.png new file mode 100644 index 0000000..4882b19 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/tour_arrow@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour/tour_arrow_down@2x.png b/marketing/resources/static/marketing/images/tour/tour_arrow_down@2x.png new file mode 100644 index 0000000..a322715 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour/tour_arrow_down@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour_phone.png b/marketing/resources/static/marketing/images/tour_phone.png new file mode 100644 index 0000000..53f6912 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour_phone.png differ diff --git a/marketing/resources/static/marketing/images/tour_phone@2x.png b/marketing/resources/static/marketing/images/tour_phone@2x.png new file mode 100644 index 0000000..09610b6 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour_phone@2x.png differ diff --git a/marketing/resources/static/marketing/images/tour_radio_buttons.png b/marketing/resources/static/marketing/images/tour_radio_buttons.png new file mode 100644 index 0000000..38d3e22 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour_radio_buttons.png differ diff --git a/marketing/resources/static/marketing/images/tour_radio_buttons_small.png b/marketing/resources/static/marketing/images/tour_radio_buttons_small.png new file mode 100644 index 0000000..38f6920 Binary files /dev/null and b/marketing/resources/static/marketing/images/tour_radio_buttons_small.png differ diff --git a/marketing/resources/static/marketing/images/viewFinder_amber-small400.png b/marketing/resources/static/marketing/images/viewFinder_amber-small400.png new file mode 100644 index 0000000..5b571d9 Binary files /dev/null and b/marketing/resources/static/marketing/images/viewFinder_amber-small400.png differ diff --git a/marketing/resources/static/marketing/images/watch_demo_header_icon@2x.png b/marketing/resources/static/marketing/images/watch_demo_header_icon@2x.png new file mode 100644 index 0000000..ed905ef Binary files /dev/null and b/marketing/resources/static/marketing/images/watch_demo_header_icon@2x.png differ diff --git a/marketing/resources/static/marketing/images/watch_demo_icon@2x.png b/marketing/resources/static/marketing/images/watch_demo_icon@2x.png new file mode 100644 index 0000000..946b85b Binary files /dev/null and b/marketing/resources/static/marketing/images/watch_demo_icon@2x.png differ diff --git a/marketing/resources/static/marketing/js/css3-mediaqueries.js b/marketing/resources/static/marketing/js/css3-mediaqueries.js new file mode 100644 index 0000000..1ea806d --- /dev/null +++ b/marketing/resources/static/marketing/js/css3-mediaqueries.js @@ -0,0 +1,779 @@ +if(typeof Object.create!=="function"){ +Object.create=function(o){ +function F(){ +}; +F.prototype=o; +return new F(); +}; +} +var ua={toString:function(){ +return navigator.userAgent; +},test:function(s){ +return this.toString().toLowerCase().indexOf(s.toLowerCase())>-1; +}}; +ua.version=(ua.toString().toLowerCase().match(/[\s\S]+(?:rv|it|ra|ie)[\/: ]([\d.]+)/)||[])[1]; +ua.webkit=ua.test("webkit"); +ua.gecko=ua.test("gecko")&&!ua.webkit; +ua.opera=ua.test("opera"); +ua.ie=ua.test("msie")&&!ua.opera; +ua.ie6=ua.ie&&document.compatMode&&typeof document.documentElement.style.maxHeight==="undefined"; +ua.ie7=ua.ie&&document.documentElement&&typeof document.documentElement.style.maxHeight!=="undefined"&&typeof XDomainRequest==="undefined"; +ua.ie8=ua.ie&&typeof XDomainRequest!=="undefined"; +var domReady=function(){ +var _1=[]; +var _2=function(){ +if(!arguments.callee.done){ +arguments.callee.done=true; +for(var i=0;i<_1.length;i++){ +_1[i](); +} +} +}; +if(document.addEventListener){ +document.addEventListener("DOMContentLoaded",_2,false); +} +if(ua.ie){ +(function(){ +try{ +document.documentElement.doScroll("left"); +} +catch(e){ +setTimeout(arguments.callee,50); +return; +} +_2(); +})(); +document.onreadystatechange=function(){ +if(document.readyState==="complete"){ +document.onreadystatechange=null; +_2(); +} +}; +} +if(ua.webkit&&document.readyState){ +(function(){ +if(document.readyState!=="loading"){ +_2(); +}else{ +setTimeout(arguments.callee,10); +} +})(); +} +window.onload=_2; +return function(fn){ +if(typeof fn==="function"){ +_1[_1.length]=fn; +} +return fn; +}; +}(); +var cssHelper=function(){ +var _3={BLOCKS:/[^\s{][^{]*\{(?:[^{}]*\{[^{}]*\}[^{}]*|[^{}]*)*\}/g,BLOCKS_INSIDE:/[^\s{][^{]*\{[^{}]*\}/g,DECLARATIONS:/[a-zA-Z\-]+[^;]*:[^;]+;/g,RELATIVE_URLS:/url\(['"]?([^\/\)'"][^:\)'"]+)['"]?\)/g,REDUNDANT_COMPONENTS:/(?:\/\*([^*\\\\]|\*(?!\/))+\*\/|@import[^;]+;)/g,REDUNDANT_WHITESPACE:/\s*(,|:|;|\{|\})\s*/g,MORE_WHITESPACE:/\s{2,}/g,FINAL_SEMICOLONS:/;\}/g,NOT_WHITESPACE:/\S+/g}; +var _4,_5=false; +var _6=[]; +var _7=function(fn){ +if(typeof fn==="function"){ +_6[_6.length]=fn; +} +}; +var _8=function(){ +for(var i=0;i<_6.length;i++){ +_6[i](_4); +} +}; +var _9={}; +var _a=function(n,v){ +if(_9[n]){ +var _b=_9[n].listeners; +if(_b){ +for(var i=0;i<_b.length;i++){ +_b[i](v); +} +} +} +}; +var _c=function(_d,_e,_f){ +if(ua.ie&&!window.XMLHttpRequest){ +window.XMLHttpRequest=function(){ +return new ActiveXObject("Microsoft.XMLHTTP"); +}; +} +if(!XMLHttpRequest){ +return ""; +} +var r=new XMLHttpRequest(); +try{ +r.open("get",_d,true); +r.setRequestHeader("X_REQUESTED_WITH","XMLHttpRequest"); +} +catch(e){ +_f(); +return; +} +var _10=false; +setTimeout(function(){ +_10=true; +},5000); +document.documentElement.style.cursor="progress"; +r.onreadystatechange=function(){ +if(r.readyState===4&&!_10){ +if(!r.status&&location.protocol==="file:"||(r.status>=200&&r.status<300)||r.status===304||navigator.userAgent.indexOf("Safari")>-1&&typeof r.status==="undefined"){ +_e(r.responseText); +}else{ +_f(); +} +document.documentElement.style.cursor=""; +r=null; +} +}; +r.send(""); +}; +var _11=function(_12){ +_12=_12.replace(_3.REDUNDANT_COMPONENTS,""); +_12=_12.replace(_3.REDUNDANT_WHITESPACE,"$1"); +_12=_12.replace(_3.MORE_WHITESPACE," "); +_12=_12.replace(_3.FINAL_SEMICOLONS,"}"); +return _12; +}; +var _13={mediaQueryList:function(s){ +var o={}; +var idx=s.indexOf("{"); +var lt=s.substring(0,idx); +s=s.substring(idx+1,s.length-1); +var mqs=[],rs=[]; +var qts=lt.toLowerCase().substring(7).split(","); +for(var i=0;i-1&&_23.href&&_23.href.length!==0&&!_23.disabled){ +_1f[_1f.length]=_23; +} +} +if(_1f.length>0){ +var c=0; +var _24=function(){ +c++; +if(c===_1f.length){ +_20(); +} +}; +var _25=function(_26){ +var _27=_26.href; +_c(_27,function(_28){ +_28=_11(_28).replace(_3.RELATIVE_URLS,"url("+_27.substring(0,_27.lastIndexOf("/"))+"/$1)"); +_26.cssHelperText=_28; +_24(); +},_24); +}; +for(i=0;i<_1f.length;i++){ +_25(_1f[i]); +} +}else{ +_20(); +} +}; +var _29={mediaQueryLists:"array",rules:"array",selectors:"object",declarations:"array",properties:"object"}; +var _2a={mediaQueryLists:null,rules:null,selectors:null,declarations:null,properties:null}; +var _2b=function(_2c,v){ +if(_2a[_2c]!==null){ +if(_29[_2c]==="array"){ +return (_2a[_2c]=_2a[_2c].concat(v)); +}else{ +var c=_2a[_2c]; +for(var n in v){ +if(v.hasOwnProperty(n)){ +if(!c[n]){ +c[n]=v[n]; +}else{ +c[n]=c[n].concat(v[n]); +} +} +} +return c; +} +} +}; +var _2d=function(_2e){ +_2a[_2e]=(_29[_2e]==="array")?[]:{}; +for(var i=0;i<_4.length;i++){ +_2b(_2e,_4[i].cssHelperParsed[_2e]); +} +return _2a[_2e]; +}; +domReady(function(){ +var els=document.body.getElementsByTagName("*"); +for(var i=0;i=_44)||(max&&_46<_44)||(!min&&!max&&_46===_44)); +}else{ +return false; +} +}else{ +return _46>0; +} +}else{ +if("device-height"===_41.substring(l-13,l)){ +_47=screen.height; +if(_42!==null){ +if(_43==="length"){ +return ((min&&_47>=_44)||(max&&_47<_44)||(!min&&!max&&_47===_44)); +}else{ +return false; +} +}else{ +return _47>0; +} +}else{ +if("width"===_41.substring(l-5,l)){ +_46=document.documentElement.clientWidth||document.body.clientWidth; +if(_42!==null){ +if(_43==="length"){ +return ((min&&_46>=_44)||(max&&_46<_44)||(!min&&!max&&_46===_44)); +}else{ +return false; +} +}else{ +return _46>0; +} +}else{ +if("height"===_41.substring(l-6,l)){ +_47=document.documentElement.clientHeight||document.body.clientHeight; +if(_42!==null){ +if(_43==="length"){ +return ((min&&_47>=_44)||(max&&_47<_44)||(!min&&!max&&_47===_44)); +}else{ +return false; +} +}else{ +return _47>0; +} +}else{ +if("device-aspect-ratio"===_41.substring(l-19,l)){ +return _43==="aspect-ratio"&&screen.width*_44[1]===screen.height*_44[0]; +}else{ +if("color-index"===_41.substring(l-11,l)){ +var _48=Math.pow(2,screen.colorDepth); +if(_42!==null){ +if(_43==="absolute"){ +return ((min&&_48>=_44)||(max&&_48<_44)||(!min&&!max&&_48===_44)); +}else{ +return false; +} +}else{ +return _48>0; +} +}else{ +if("color"===_41.substring(l-5,l)){ +var _49=screen.colorDepth; +if(_42!==null){ +if(_43==="absolute"){ +return ((min&&_49>=_44)||(max&&_49<_44)||(!min&&!max&&_49===_44)); +}else{ +return false; +} +}else{ +return _49>0; +} +}else{ +if("resolution"===_41.substring(l-10,l)){ +var res; +if(_45==="dpcm"){ +res=_3d("1cm"); +}else{ +res=_3d("1in"); +} +if(_42!==null){ +if(_43==="resolution"){ +return ((min&&res>=_44)||(max&&res<_44)||(!min&&!max&&res===_44)); +}else{ +return false; +} +}else{ +return res>0; +} +}else{ +return false; +} +} +} +} +} +} +} +} +}; +var _4a=function(mq){ +var _4b=mq.getValid(); +var _4c=mq.getExpressions(); +var l=_4c.length; +if(l>0){ +for(var i=0;i0){ +s[c++]=","; +} +s[c++]=n; +} +} +if(s.length>0){ +_39[_39.length]=cssHelper.addStyle("@media "+s.join("")+"{"+mql.getCssText()+"}",false); +} +}; +var _4e=function(_4f){ +for(var i=0;i<_4f.length;i++){ +_4d(_4f[i]); +} +if(ua.ie){ +document.documentElement.style.display="block"; +setTimeout(function(){ +document.documentElement.style.display=""; +},0); +setTimeout(function(){ +cssHelper.broadcast("cssMediaQueriesTested"); +},100); +}else{ +cssHelper.broadcast("cssMediaQueriesTested"); +} +}; +var _50=function(){ +for(var i=0;i<_39.length;i++){ +cssHelper.removeStyle(_39[i]); +} +_39=[]; +cssHelper.mediaQueryLists(_4e); +}; +var _51=0; +var _52=function(){ +var _53=cssHelper.getViewportWidth(); +var _54=cssHelper.getViewportHeight(); +if(ua.ie){ +var el=document.createElement("div"); +el.style.position="absolute"; +el.style.top="-9999em"; +el.style.overflow="scroll"; +document.body.appendChild(el); +_51=el.offsetWidth-el.clientWidth; +document.body.removeChild(el); +} +var _55; +var _56=function(){ +var vpw=cssHelper.getViewportWidth(); +var vph=cssHelper.getViewportHeight(); +if(Math.abs(vpw-_53)>_51||Math.abs(vph-_54)>_51){ +_53=vpw; +_54=vph; +clearTimeout(_55); +_55=setTimeout(function(){ +if(!_3a()){ +_50(); +}else{ +cssHelper.broadcast("cssMediaQueriesTested"); +} +},500); +} +}; +window.onresize=function(){ +var x=window.onresize||function(){ +}; +return function(){ +x(); +_56(); +}; +}(); +}; +var _57=document.documentElement; +_57.style.marginLeft="-32767px"; +setTimeout(function(){ +_57.style.marginTop=""; +},20000); +return function(){ +if(!_3a()){ +cssHelper.addListener("newStyleParsed",function(el){ +_4e(el.cssHelperParsed.mediaQueryLists); +}); +cssHelper.addListener("cssMediaQueriesTested",function(){ +if(ua.ie){ +_57.style.width="1px"; +} +setTimeout(function(){ +_57.style.width=""; +_57.style.marginLeft=""; +},0); +cssHelper.removeListener("cssMediaQueriesTested",arguments.callee); +}); +_3c(); +_50(); +}else{ +_57.style.marginLeft=""; +} +_52(); +}; +}()); +try{ +document.execCommand("BackgroundImageCache",false,true); +} +catch(e){ +} + diff --git a/marketing/resources/static/marketing/js/global.js b/marketing/resources/static/marketing/js/global.js new file mode 100644 index 0000000..f502a5e --- /dev/null +++ b/marketing/resources/static/marketing/js/global.js @@ -0,0 +1,891 @@ +var g_nViewportWid = $(window).width(); +var g_nViewportHei = $(window).height(); +var g_nBreakpointWid = parseInt($("#css_vp_wid").css("width"), 10); +//var g_nScrollTop = $(window).scrollTop(); +//var g_nScrollBot = g_nScrollTop + g_nViewportHei; + + +var g_xVF = { + debug:0, + homepage_slides: [ + "/static/marketing/images/pemberton_viewFinder.coHero5.jpg", + "/static/marketing/images/V1-1_Website_HeroImage_2.png", + "/static/marketing/images/pemberton_viewFinder.coHero1.jpg", + "/static/marketing/images/V1-1_Website_HeroImage_1.png", + "/static/marketing/images/pemberton_viewFinder.coHero3.jpg" + ], + homepage_slideshow_playing: false, + current_homepage_slide: 0, + tour_player: false, + current_tour_video: 1, + has_html5_video: Modernizr.video && !Modernizr.touch, + dial_player: false, + dial_played: false, + current_page: "", + tour_convo_data: { + current_phone_index:null, + positions: [] + }, + + + + init: function(){ + _eventAdd("resize", this.onResize, this); + _eventAdd("scroll", this.onScroll, this); + + var self = this; + self.current_page = $("body").attr("data-current-page"); + + if(self.current_page=="homepage"){ + self.initHomepage(); + self.initTour(); + } + + else if(self.current_page=="faq"){ + self.initFaq(); + } + + + $(window).resize(function(){ + _eventFire("resize"); + }); + + setTimeout(function(){ + _eventFire("resize"); + },1); + + $(window).scroll(function(){ + _eventFire("scroll"); + }); + + _eventFire("scroll"); + }, + + initHomepage: function(){ + var self = this; + + if(self.has_html5_video){ + $("#dial_panel").addClass("has_video"); + videojs("dial_bkgd_video").ready(function(){ + self.dial_player = this; + self.dial_player.volume(0); + }); + /* + videojs("tour_video").ready(function(){ + self.tour_player = this; + self.tour_player.volume(0); + self.tour_player.on("ended", function(){ + _eventFire("play_tour_video", { num:++self.current_tour_video }); + }); + }); + */ + } + else { + $("#dial_bkgd_video").remove(); + //$("#tour_video").remove(); + } + self.current_homepage_slide = Math.floor(Math.random()*self.homepage_slides.length); + //self.homepage_slides.shuffle(); + + self.startHomepageSlideshow(); + }, + + initFaq: function(){ + + $(".faq-toc").on("click", "li a", function(e){ + e.preventDefault(); + var toc_num = $(this).parent().index(); + var faq_pos = $(".faq-list li.faq-item").eq(toc_num).offset().top; + location.href = '#'+(toc_num+1); + $("html, body").animate({"scrollTop" : faq_pos }); + }); + if(window.location.hash){ + var hash = parseInt(location.hash.substr(1)); + if(!isNaN(hash)) { + var toc_num = Math.max(0, hash-1); + $(".faq-toc li.faq-item").eq(toc_num).find("a").trigger("click"); + } + } + }, + + initTour: function(){ + var tour_intro = $("#tour_intro_panel"); + var self = this; + + var convo_panel = $("#conversation_panel"); + }, + + tourIntroOpen: function(side, animate){ + var self = this; + var tour_intro = $("#tour_intro_panel"); + + if(self.tourIntroSliding) return; + + + var tour_phone_inbox = $(".tour_iphone_inbox"); + var tour_phone_library = $(".tour_iphone_library"); + var detail_inbox = tour_intro.find(".detail_inbox"); + var detail_library = tour_intro.find(".detail_library"); + var tour_intro_copy = tour_intro.find(".intro_copy"); + var tour_intro_frame = tour_intro.find(".content_frame"); + + self.tourIntroSliding = true; + tour_intro.addClass("focused"); + + if(g_nBreakpointWid <= 1000){ + + //tour_intro.animate({ "height":"1000px" }, 1000, "easeInOutQuart"); + } + else { + tour_intro.find(".intro_copy").css({ "position":"absolute", "width":tour_intro_copy.width()+"px","top":tour_intro_copy.position().top+"px","left":tour_intro_copy.position().left+"px" });//.fadeOut(400); + + if(animate){ + tour_intro.find(".caption_holder").fadeOut(400); + tour_intro.animate({ "padding-bottom":"60px" }, 1000, "easeInOutQuart"); + } + else { + tour_intro.find(".caption_holder").hide(); + tour_intro.css({ "padding-bottom":"60px" }); + } + } + + // inbox click (right) + if(side=="inbox"){ + if(g_nBreakpointWid <= 600){ + var detail_inbox_top = (tour_intro_frame.find(".divider").position().top + tour_intro_frame.find(".divider").height()) + 50 + 430 + 44; // margin-top + height (.tour_iphone_library) + if(animate){ + tour_phone_inbox.find(".caption_holder").animate({ top:"10px", opacity:0 }, 800, "easeInOutQuart"); + detail_inbox.show().css({ opacity:0 }); + detail_inbox.animate({ top:detail_inbox_top+"px", opacity:1 }, 1000, "easeInOutQuart", function(){ + self.tourIntroSliding = false; + }); + tour_phone_inbox.animate({ "margin-top":(tour_intro.find(".detail_inbox").height()+44)+"px", "padding-top":0, height:"570px" }, 800, "easeInOutQuart"); + + tour_phone_library.animate({ "margin-top":"50px", "padding-top":"180px", height:"430px" }, 800, "easeInOutQuart"); + tour_intro.find(".tour_iphone_library .caption_holder").animate({ top:0, opacity:1 }, 800, "easeInOutQuart"); + detail_library.fadeOut(400); + } + else { + tour_phone_inbox.find(".caption_holder").css({ top:"10px", opacity:0 }); + detail_inbox.show().css({ top:detail_inbox_top+"px", opacity:1 }); + self.tourIntroSliding = false; + tour_phone_inbox.css({ "margin-top":(tour_intro.find(".detail_inbox").height()+44)+"px", "padding-top":0, height:"570px" }); + tour_phone_library.css({ "margin-top":"50px", "padding-top":"180px", height:"430px" }); + tour_phone_library.find(".caption_holder").css({ top:0, opacity:1 }); + detail_library.hide(); + } + } + else if(g_nBreakpointWid <= 1000){ + var detail_inbox_right = tour_intro_frame.width()+ 40 - tour_intro.find(".detail_inbox").width(); + if((tour_phone_library.offset().left) < 0){ + detail_inbox_right = (tour_intro_frame.offset().left + tour_intro_frame.width()) - tour_intro.find(".detail_inbox").width() - 40; + } + if(animate){ + tour_phone_inbox.animate({ "margin-top":"0px" }, 800, "easeInOutQuart"); + tour_phone_inbox.find(".caption_holder").animate({ "top":"250px" }, 800, "easeInOutQuart"); + tour_phone_library.delay(300).animate({ "margin-top":"550px" }, 1000, "easeInOutQuart"); + tour_phone_library.find(".caption_holder").delay(300).animate({ "top":"270px" }, 1000, "easeInOutQuart"); + detail_library.animate({ left:"-60px", opacity:0 }, 800, "easeInOutQuart", function(){ + $(this).hide(); + self.tourIntroSliding = false; + }); + detail_inbox.show().delay(300).animate({ right:detail_inbox_right+"px", opacity:1 }, 1000, "easeInOutQuart"); + } + else { + tour_phone_inbox.css({ "margin-top":"0px" }); + tour_phone_inbox.find(".caption_holder").css({ "top":"250px" }); + tour_phone_library.css({ "margin-top":"550px" }); + tour_phone_library.find(".caption_holder").css({ "top":"270px" }); + detail_library.css({ left:"-60px", opacity:0 }).hide(); + self.tourIntroSliding = false; + detail_inbox.show().css({ right:detail_inbox_right+"px", opacity:1 }); + } + + } + else { + var inbox_right_margin = tour_intro.find(".content_frame").width() - ( tour_intro.find(".content_frame").width()*0.15) - tour_intro.find(".tour_iphone_inbox").width(); + if(animate){ + tour_phone_library.delay(200).animate({ "margin-left":"-260px" }, 1000, "easeInOutQuart"); + tour_phone_inbox.delay(350).animate({ "margin-right":Math.round(inbox_right_margin)+"px" }, { + duration:1000, + easing:"easeInOutQuart", + step: function(now, tween){ + self.tourIntroMotionStep(); + }, + complete: function(){ + self.tourIntroSliding = false; + } + }); + } + else { + tour_phone_library.css({ "margin-left":"-260px" }); + tour_phone_inbox.css({ "margin-right":Math.round(inbox_right_margin)+"px" }); + tour_intro_copy.find(".copy_mask").css({ left:"auto", right:0, width:tour_intro_copy.width()+"px" }); + self.tourIntroMotionStep(); + self.tourIntroSliding = false; + } + } + + } + // library click (left) + else { + if(g_nBreakpointWid <= 600){ + var detail_library_top = tour_intro_frame.find(".divider").position().top + tour_intro_frame.find(".divider").height() + 40; + if(animate){ + tour_phone_library.find(".caption_holder").animate({ top:"10px", opacity:0 }, 800, "easeInOutQuart"); + detail_library.show().css({ opacity:0 }); + detail_library.animate({ top:detail_library_top+"px", opacity:1 }, 1000, "easeInOutQuart", function(){ + self.tourIntroSliding = false; + }); + tour_phone_library.animate({ "margin-top":(tour_intro.find(".detail_library").height()+44)+"px", "padding-top":0, height:"570px" }, 800, "easeInOutQuart"); + tour_phone_inbox.animate({ "margin-top":"50px", "padding-top":"180px", height:"430px" }, 800, "easeInOutQuart"); + tour_phone_inbox.find(".caption_holder").animate({ top:0, opacity:1 }, 800, "easeInOutQuart"); + detail_inbox.fadeOut(400); + } + else { + tour_phone_library.find(".caption_holder").css({ top:"10px", opacity:0 }); + detail_library.show().css({ top:detail_library_top+"px", opacity:1 }); + self.tourIntroSliding = false; + tour_phone_library.css({ "margin-top":(tour_intro.find(".detail_library").height()+44)+"px", "padding-top":0, height:"570px" }); + tour_phone_inbox.css({ "margin-top":"50px", "padding-top":"180px", height:"430px" }); + tour_phone_inbox.find(".caption_holder").css({ top:0, opacity:1 }); + detail_inbox.hide(); + } + } + else if(g_nBreakpointWid <= 1000){ + var detail_library_left = (tour_phone_inbox.offset().left - tour_intro_frame.offset().left) + 40; + if((tour_phone_library.offset().left) < 0){ + detail_library_left = (tour_intro_frame.offset().left + tour_intro_frame.width()) - detail_library.width() - 40; + } + if(animate){ + tour_phone_library.animate({ "margin-top":"0px" }, 800, "easeInOutQuart"); + tour_phone_library.find(".caption_holder").animate({ "top":"250px" }, 800, "easeInOutQuart"); + tour_phone_inbox.delay(300).animate({ "margin-top":"550px" }, 1000, "easeInOutQuart"); + tour_phone_inbox.find(".caption_holder").delay(300).animate({ "top":"270px" }, 1000, "easeInOutQuart"); + detail_inbox.animate({ right:"-60px", opacity:0 }, 800, "easeInOutQuart", function(){ + $(this).hide(); + self.tourIntroSliding = false; + }); + detail_library.show().delay(300).animate({ left:detail_library_left+"px", opacity:1 }, 1000, "easeInOutQuart"); + } + else { + tour_phone_library.css({ "margin-top":"0px" }); + tour_phone_library.find(".caption_holder").css({ "top":"250px" }); + tour_phone_inbox.css({ "margin-top":"550px" }); + tour_phone_inbox.find(".caption_holder").css({ "top":"270px" }); + detail_inbox.css({ right:"-60px", opacity:0 }).hide(); + self.tourIntroSliding = false; + detail_library.show().css({ left:detail_library_left+"px", opacity:1 }); + } + } + else { + var inbox_right_margin = (tour_intro.find(".content_frame").width()*0.85) - tour_intro.find(".tour_iphone_inbox").width(); + if(animate){ + tour_phone_inbox.delay(200).animate({ "margin-right":"-260px" }, 1000, "easeInOutQuart"); + tour_phone_library.delay(350).animate({ "margin-left":inbox_right_margin+"px" }, { + duration:1000, + easing: "easeInOutQuart", + step:function(){ + self.tourIntroMotionStep(); + }, + complete: function(){ + self.tourIntroSliding = false; + } + }); + } + else { + tour_phone_inbox.css({ "margin-right":"-260px" }); + tour_phone_library.css({ "margin-left":inbox_right_margin+"px" }); + tour_intro_copy.find(".copy_mask").css({ left:0, right:'auto', width:tour_intro_copy.width()+"px" }); + self.tourIntroMotionStep(); + self.tourIntroSliding = false; + } + } + } + + if(g_nBreakpointWid > 1000 && animate){ + $("html, body").stop(true).animate({scrollTop: tour_intro.offset().top }, 1000, 'easeInOutQuint'); + } + + tour_intro.find(".iphone_holder").not($(".tour_iphone_"+side)).removeClass("disabled"); + $(".tour_iphone_"+side).addClass("disabled"); + + }, + + tourIntroClose: function(){ + var self = this; + var tour_intro = $("#tour_intro_panel"); + + self.tourIntroSliding = true; + tour_intro.find(".iphone_holder").removeClass("disabled"); + + if(g_nBreakpointWid <= 600){ + tour_intro.find(".iphone_holder").animate({ "margin-top":"50px", "padding-top":"180px", height:"430px" }, 800, "easeInOutQuart"); + tour_intro.find(".iphone_holder .caption_holder").animate({ top:0, opacity:1 }, 800, "easeInOutQuart", function(){ + self.tourIntroSliding = false; + }); + tour_intro.find(".detail_holder").fadeOut(400); + } + + else if(g_nBreakpointWid <= 1000){ + tour_intro.find(".detail_inbox").animate({ right:"-60px", opacity:0 }, 800, "easeInOutQuart"); + tour_intro.find(".tour_iphone_inbox").delay(200).animate({ "margin-top":"250px" }, 800, "easeInOutQuart"); + tour_intro.find(".tour_iphone_inbox .caption_holder").delay(200).animate({ "top":"-10px" }, 800, "easeInOutQuart"); + + tour_intro.find(".detail_library").animate({ left:"-60px", opacity:0 }, 800, "easeInOutQuart"); + tour_intro.find(".tour_iphone_library").delay(200).animate({ "margin-top":"250px" }, 800, "easeInOutQuart"); + tour_intro.find(".tour_iphone_library .caption_holder").delay(200).animate({ "top":"-10px" }, 800, "easeInOutQuart", function(){ + self.tourIntroSliding = false; + _eventFire("resize"); + }); + + } + else { + tour_intro.find(".tour_iphone_library").animate({ "margin-left":"-75px" }, { + duration:1000, + easing: "easeInOutQuart", + step:function(){ + self.tourIntroMotionStep(); + } + }); + + tour_intro.find(".tour_iphone_inbox").animate({ "margin-right":"-75px" }, { + duration:1000, + easing: "easeInOutQuart", + step:function(){ + //self.tourIntroMotionStep(); + } + }); + tour_intro.animate({ "padding-bottom":"300px" }, 1000, 'easeInOutQuint', function(){ + self.tourStartMagnetism = true; + self.tourIntroSliding = false; + //$(this).removeAttr("style"); + _eventFire("resize"); + }); + tour_intro.find(".caption_holder").fadeIn(400); + } + }, + + + tourConvoSlidePhone: function(ind, animate){ + var self = this; + if(self.tourConvoSliding) return; + //if(self.tour_convo_data.current_phone_index==ind) return; + + var convo_panel = $("#conversation_panel"); + var convo_phone_list = convo_panel.find(".phone_list"); + var convo_phones = convo_panel.find(".phone_list li"); + + var dir = (ind > self.tour_convo_data.current_phone_index || (ind==0 && self.tour_convo_data.current_phone_index==convo_phones.length-1)) ? "left":"right"; + if(self.tour_convo_data.current_phone_index==0 && ind==convo_phones.length-1) dir = "right"; + self.tour_convo_data.current_phone_index = ind; + + // radio active states + convo_panel.find(".convo_nav_list li.radio").removeClass("current"); + convo_panel.find(".convo_nav_list li.radio").eq(ind).addClass("current"); + + var current_caption = convo_panel.find(".caption .copy.current"); + var goto_caption = $("#convo_caption"+ind);//convo_panel.find(".caption .copy.current").eq(ind); + goto_caption.addClass("current"); + + // set current class to phone + convo_phones.removeClass("current"); + var convo_focused_phone = $("#convo_phone"+ind); + convo_focused_phone.addClass("current"); + var focused_phone_ind = convo_phones.index(convo_focused_phone); + + // set position index + convo_phones.each(function(i, el){ + $(el).attr("data-prev-index", $(el).attr("data-index")); + + if(i > focused_phone_ind) { + if(focused_phone_ind==0 && i==convo_phones.length-1) $(el).attr("data-index", 0); + else $(el).attr("data-index", (convo_phones.index($(el)) - focused_phone_ind)+1); + } + if(i < focused_phone_ind){ + if(i==focused_phone_ind-1) $(el).attr("data-index", 0); + else $(el).attr("data-index", ((convo_phones.length - focused_phone_ind) + convo_phones.index($(el)))+1); + } + }); + + convo_focused_phone.attr("data-index", 1); + + current_caption.hide(); + goto_caption.show(); + + if(g_nBreakpointWid > 600 && g_nBreakpointWid <= 1000){ + convo_phones.hide(); + $("#convo_phone"+ind).show(); + } + else { + + convo_phones.each(function(i, el){ + var pos_ind = parseInt($(el).attr("data-index")); + var pos_prev_ind = parseInt($(el).attr("data-prev-index")); + var pos_left = self.tour_convo_data.positions[pos_ind]; + + if(!animate){ + $(el).css({ left:pos_left+"px" }); + return; + } + self.tourConvoSliding = true; + + var off_screen_left = convo_panel.find(".phone_list").offset().left + 382; + var off_screen_right = g_nViewportWid-convo_panel.find(".phone_list").offset().left; + + // slide first left, loop + if(dir=="left" && pos_prev_ind < pos_ind){ //(pos_prev_ind==0 && pos_ind==convo_phones.length-1) || ){ + $(el).stop(true).animate({ left:-off_screen_left+"px" }, 400, 'easeInOutQuint', function(){ + $(this).css({ left:off_screen_right+"px" }).animate({ left:pos_left+"px" }, 300, 'easeOutQuint'); + self.tourConvoSliding = false; + }); + } + // slide last right, loop + else if(dir=="right" && pos_prev_ind > pos_ind){ //pos_prev_ind==convo_phones.length-1 && pos_ind==0){ + $(el).stop(true).animate({ left:off_screen_right+"px" }, 400, 'easeInOutQuint', function(){ + $(this).css({ left:-off_screen_left+"px" }).animate({ left:pos_left+"px" }, 300, 'easeOutQuint'); + self.tourConvoSliding = false; + }); + } + else { + // slide left, then loop + var params = { + duration:500, + easing:'easeInOutQuint', + complete: function(){ + self.tourConvoSliding = false; + } + }; + $(el).stop(true).animate({ left:pos_left+"px" }, params); + } + + }); + + } + }, + + tourIntroMotionStep: function(){ + var tour_intro = $("#tour_intro_panel"); + var tour_intro_copy = tour_intro.find(".intro_copy"); + var tour_intro_copy_mask = tour_intro.find(".intro_copy .copy_mask"); + tour_intro.find(".detail_holder").css({ 'display':'inline' }); + + var detail_holder_inbox = tour_intro.find(".detail_inbox"); + var tour_iphone_inbox = tour_intro.find(".tour_iphone_inbox"); + var tour_iphone_inbox_left = tour_iphone_inbox.offset().left; + var tour_iphone_inbox_right = tour_iphone_inbox.offset().left + tour_iphone_inbox.width(); + + var detail_holder_library = tour_intro.find(".detail_library"); + var tour_iphone_library = tour_intro.find(".tour_iphone_library"); + var tour_iphone_library_left = tour_iphone_library.offset().left; + var tour_iphone_library_right = tour_iphone_library.offset().left + tour_iphone_library.width(); + + var copy_mask_wid = 0; + var detail_library_mask_wid = detail_holder_library.width(); + var detail_inbox_mask_wid = detail_holder_inbox.width(); + + if(tour_iphone_library_right > tour_intro_copy.offset().left){ + copy_mask_wid = (tour_iphone_library_left - tour_intro_copy_mask.offset().left) + 63; + tour_intro_copy_mask.css({ left:0, right:'auto', width:copy_mask_wid+"px" }); + } + if(tour_iphone_inbox_left < (tour_intro_copy.offset().left+tour_intro_copy.width())){ + copy_mask_wid = (tour_intro_copy_mask.offset().left + tour_intro_copy_mask.width()) - tour_iphone_inbox_left -33; + tour_intro_copy_mask.css({ left:"auto", right:0, width:copy_mask_wid+"px" }); + } + + if(tour_iphone_library_left+33 > detail_holder_library.offset().left){ + detail_library_mask_wid = detail_holder_library.width() - ((tour_iphone_library_left+33) - detail_holder_library.offset().left); + detail_holder_library.css({ opacity:1 }); + } + else detail_holder_library.css({ opacity:0 }); + + detail_holder_library.find(".detail_mask").css({ width:detail_library_mask_wid+"px" }); + + if(tour_iphone_inbox_right < detail_holder_inbox.offset().left+detail_holder_inbox.width()){ + detail_inbox_mask_wid = (tour_iphone_inbox_left + tour_iphone_inbox.width()) - detail_holder_inbox.offset().left-33; + detail_holder_inbox.css({ opacity:1 }); + } + else detail_holder_inbox.css({ opacity:0 }); + + detail_holder_inbox.find(".detail_mask").css({ width:detail_inbox_mask_wid+"px" }); + + + }, + + tourOpenWatchVideoOverlay: function(){ + var self = this; + self.youtube_player_loaded = false; + + $("#dial_video_holder").html(self.youtube_video_embed); + + demo_video = new YT.Player('dial_video_iframe', { + height: '315', + width: '560', + videoId: '1IH6FZlzTRY', + playerVars: { 'autoplay': 0 }, + events: { + 'onReady': function(e){ + self.youtube_player_loaded = true; + if(self.youtube_start_video_onload){ + demo_video.playVideo(); + } + }, + 'onStateChange': function (e){ + if(e.data==0){ + self.tourCloseWatchVideoOverlay(); + } + } + } + }); + + $("#tour_video_overlay").fadeIn(500); + + self.adjustDialVideoPos(); + if(!Modernizr.touch) { + if(demo_video && self.youtube_player_loaded) { + demo_video.playVideo(); + } + else self.youtube_start_video_onload = true; + } + }, + + tourCloseWatchVideoOverlay: function(){ + var self = this; + $("#tour_video_overlay").fadeOut(500, function(){ + self.youtube_player_loaded = false; + self.youtube_start_video_onload = false; + $("#dial_video_iframe").remove(); + }); + if(demo_video) { + demo_video.pauseVideo(); + demo_video.destroy(); + } + }, + + startHomepageSlideshow: function(){ + var self = this; + + self.homepage_slideshow_playing = true; + self.current_homepage_slide++; + if(self.current_homepage_slide >= self.homepage_slides.length){ + self.current_homepage_slide = 0; + } + var next_img = self.homepage_slides[self.current_homepage_slide]; + + $("#home_hero .bkgd_img_off").css({ "background-image":"url("+next_img+")" }).fadeIn(1000, function(){ + $(this).removeClass("bkgd_img_off").addClass("bkgd_img"); + }); + $("#home_hero .bkgd_img").fadeOut(1000, function(){ + $(this).removeClass("bkgd_img").addClass("bkgd_img_off"); + + $("#animate_ref").animate({ opacity:1 }, 5000, function(){ + $(this).css({ opacity:0 }); + self.startHomepageSlideshow(); + }); + }); + + }, + + pauseHomepageSlideshow: function(){ + var self = this; + self.homepage_slideshow_playing = false; + $("#animate_ref").stop(); + }, + + adjustDialVideoPos: function(){ + var viewport_ratio = g_nViewportWid/g_nViewportHei; + var video_ratio = $("#dial_video_holder").attr("data-ratio"); + + var video_wid = g_nViewportWid-25; + var video_hei = video_wid/video_ratio; + + if(viewport_ratio > video_ratio){ + video_hei = g_nViewportHei-25; + video_wid = video_hei * video_ratio; + } + $("#dial_video_holder").css({ "width":video_wid+"px", "height":video_hei+"px" }); + $("#dial_video_holder iframe").attr("width", video_wid).attr("height", video_hei); + + var video_top = Math.max(0, (g_nViewportHei - video_hei)/2); + var video_left = Math.max(0, (g_nViewportWid - video_wid)/2); + + $("#dial_video_holder").css({ 'margin-top':video_top+"px", "margin-left":video_left+"px" }); + + }, + + onExpandDialVideo: function(e, params){ + if($("#dial_panel").hasClass("expanded")) return; + + //$.address.state("/video/"); + var self = e.data; + + var panel = $("#dial_panel"); + panel.find(".bkgd_img").fadeOut(400); + panel.find(".content_frame").fadeOut(400); + panel.animate({ height:g_nViewportHei+"px", "backgroundColor":"#000" }, 1000, 'easeInOutQuint', function(){ + self.adjustDialVideoPos(); + + panel.addClass("expanded").find(".close").fadeIn(400); + $("#dial_video_holder").fadeIn(400); + + if(demo_video && !Modernizr.touch) demo_video.playVideo(); + }); + + $("html, body").stop(true).animate({scrollTop: $("#dial_panel").offset().top }, 1000, 'easeInOutQuint'); + + }, + + onCollapseDialVideo: function(e, params){ + var panel = $("#dial_panel"); + var orig_height = panel.find(".bkgd_img").height(); + + if(demo_video) demo_video.pauseVideo(); + + panel.find(".close").fadeOut(400); + $("#dial_video_holder").fadeOut(400); + panel.animate({ height:orig_height+"px" }, 1000, 'easeInOutQuint', function(){ + panel.removeClass("expanded"); + panel.removeAttr("style"); + }); + panel.find(".content_frame").delay(700).fadeIn(400); + panel.find(".bkgd_img").delay(700).fadeIn(400); + + $("html, body").stop(true).animate({scrollTop: ($("#dial_panel").offset().top - 50) }, 1000, 'easeInOutQuint'); + + }, + + onPlayTourVideo: function(e, params){ + var self = e.data; + + self.current_tour_video = params.num; + + //if(!self.tour_player) return; + if(self.current_tour_video > 3) self.current_tour_video = 1; + + + $("#tour_points .tour_point").removeClass("current"); + $("#tour_video_holder").removeClass("point_1 point_2 point_3"); + + var vid_mp4 = "/static/marketing/video/FILE0059_converted.mp4"; + var vid_webm = "/static/marketing/video/FILE0059.webm"; + var vid_ogg = "/static/marketing/video/FILE0059.ogv"; + + if(self.current_tour_video==1){ + vid_mp4 = "/static/marketing/video/FILE0063_converted.mp4"; + vid_webm = "/static/marketing/video/FILE0063.webm"; + vid_ogg = "/static/marketing/video/FILE0063.ogv"; + + $("#tour_video_holder").addClass("point_1"); + $(".tour_point_1").addClass("current"); + } + else if(self.current_tour_video==2){ + vid_mp4 = "/static/marketing/video/FILE0022_converted.mp4"; + vid_webm = "/static/marketing/video/FILE0022.webm"; + vid_ogg = "/static/marketing/video/FILE0022.ogv"; + + $("#tour_video_holder").addClass("point_2"); + $(".tour_point_2").addClass("current"); + } + else { + $("#tour_video_holder").addClass("point_3"); + $(".tour_point_3").addClass("current"); + } + + /* + if(self.has_html5_video && !self.tour_player){ + self.tour_player.src([ + { type: "video/mp4", src: vid_mp4 }, + { type: "video/webm", src: vid_webm }, + { type: "video/ogg", src: vid_ogg } + ]).play(); + } + */ + }, + + onResize: function(e, params){ + var self = e.data; + + g_nViewportWid = $(window).width(); + g_nViewportHei = $(window).height(); + g_nBreakpointWid = self.getBreakpointWid(); + + if(self.debug) + $("#debug").html(g_nViewportWid+" ("+g_nBreakpointWid+") x "+g_nViewportHei).show(); + + if(self.current_page=="homepage"){ + self.resizeHomepage(e); + self.resizeTour(e); + } + }, + + resizeHomepage:function(e){ + + var self = e.data; + + $("#download_panel .download").addClass("large"); + + if(g_nBreakpointWid <= 1100){ + if(g_nBreakpointWid <= 600){ + $("#download_panel .download").removeClass("large"); + } + if(self.has_html5_video){ + if(self.dial_player) self.dial_player.dimensions(924,373); + //if(self.tour_player) self.tour_player.dimensions(201,357); + } + } + else { + if(self.has_html5_video){ + if(self.dial_player) self.dial_player.dimensions(1200,484); + //if(self.tour_player) self.tour_player.dimensions(268,476); + } + } + + ("#dial_bkgd_video video").attr("width", "100%"); + + if($("#dial_panel").hasClass("expanded")){ + $("#dial_panel").css({ "height":g_nViewportHei+"px" }); + self.adjustDialVideoPos(); + } + }, + + resizeTour:function(e){ + + var self = e.data; + + var convo_panel = $("#conversation_panel"); + var phone_list = $("#conversation_panel .phone_list"); + var tour_panel = $("#tour_intro_panel"); + var tour_panel_intro_copy = tour_panel.find(".intro_copy"); + + //phone_list.find(".phone3").removeAttr("style"); + //tour_panel_intro_copy.removeAttr("style"); + $(".resize_clear").removeAttr("style"); + $(".imgwid_100").removeAttr("height").attr("width", "100%"); + + //tour_panel.find(".iphone_holder").removeClass("disabled"); + //tour_panel.removeClass("focused"); + + + if(g_nBreakpointWid <= 600){ + var detail_left = (tour_panel.find(".content_frame").width() - tour_panel.find(".detail_holder").width())/2; + tour_panel.find(".detail_holder").css({ left:detail_left+"px" }); + + var max_copy_hei = 0; + convo_panel.find(".caption .copy").each(function(i, el){ + max_copy_hei = Math.max(max_copy_hei, $(el).show().height()); + $(el).hide(); + }); + convo_panel.find(".caption").css({ height:max_copy_hei+"px" }); + + + var convo_panel_frame_wid = convo_panel.width(); + var convo_panel_phone_wid = phone_list.find("li").eq(0).width(); + self.tour_convo_data.positions = new Array(); + self.tour_convo_data.positions[1] = (convo_panel_frame_wid - convo_panel_phone_wid)/2; + self.tour_convo_data.positions[0] = self.tour_convo_data.positions[1] - convo_panel_phone_wid - 22; + self.tour_convo_data.positions[2] = self.tour_convo_data.positions[1] + convo_panel_phone_wid + 22; + self.tour_convo_data.positions[3] = self.tour_convo_data.positions[2] + convo_panel_phone_wid + 22; + self.tour_convo_data.positions[4] = self.tour_convo_data.positions[3] + convo_panel_phone_wid + 22; + for(var p in self.tour_convo_data.positions){ + phone_list.find(".phone"+p).css({ "left":(self.tour_convo_data.positions[p])+"px" }); + } + } + else if(g_nBreakpointWid <= 1000){ + var intro_phones_wid = (tour_panel.find(".tour_iphone_inbox").offset().left + tour_panel.find(".tour_iphone_inbox").width()) - tour_panel.find(".tour_iphone_library").offset().left; + if(g_nViewportWid < intro_phones_wid){ + var width_diff = (intro_phones_wid - g_nViewportWid)/2; + tour_panel.find(".tour_iphone_library .caption_holder").css({ "left":width_diff+"px" }); + tour_panel.find(".tour_iphone_inbox .caption_holder").css({ "right":width_diff+"px" }); + } + } + else { + // tour intro + // freeze intro_copy #magnetism + tour_panel_intro_copy.css({ left:((tour_panel.find(".content_frame").width() - tour_panel_intro_copy.width())/2)+"px" }); + + // conversation positioning + var convo_panel_frame_wid = convo_panel.find(".content_frame").width(); + self.tour_convo_data.positions = new Array(); + self.tour_convo_data.positions[0] = (-convo_panel_frame_wid*0.15); + self.tour_convo_data.positions[1] = (convo_panel_frame_wid*0.55); + self.tour_convo_data.positions[2] = Math.max(self.tour_convo_data.positions[1]+370, (convo_panel_frame_wid*0.85)); + self.tour_convo_data.positions[3] = Math.max(self.tour_convo_data.positions[2]+370, (convo_panel_frame_wid*1.15)); + self.tour_convo_data.positions[4] = Math.max(self.tour_convo_data.positions[3]+370, (convo_panel_frame_wid*1.45)); + for(var p in self.tour_convo_data.positions){ + phone_list.find(".phone"+p).css({ "left":(self.tour_convo_data.positions[p])+"px" }); + } + } + + var current_convo_ind = 0; + if(phone_list.find("li.current").length > 0) + current_convo_ind = phone_list.find("li").index(phone_list.find("li.current")); + self.tourConvoSlidePhone(current_convo_ind, false); + + if(tour_panel.find(".iphone_holder.disabled").length > 0){ + var side = "library"; + if(tour_panel.find(".iphone_holder.disabled").hasClass("tour_iphone_inbox")) + side = "inbox"; + + self.tourIntroOpen(side, false); + } + + // set some helper data vars + /* + tour_panel.find(".iphone_holder").each(function(i, el){ + $(el).attr("data-off-x", $(el).position().left).attr("data-off-y", $(el).position().top); + }); + */ + + }, + + + onScroll: function(e, params){ + g_nScrollTop = $(window).scrollTop(); + g_nScrollBot = g_nScrollTop + g_nViewportHei; + + var self = e.data; + + if(self.current_page=="homepage"){ + if(($("#dial_panel").offset().top+$("#dial_panel").height()) < g_nScrollBot) { + if(self.dial_player && !self.dial_played){ + self.dial_played = true; + self.dial_player.play(); + } + } + + // stop hero slideshow if off page + if($("#home_hero").in_viewport()){ + if(!self.homepage_slideshow_playing) + self.startHomepageSlideshow(); + } + else { + self.pauseHomepageSlideshow(); + } + + // stop tour video if off page + if(self.has_html5_video){ + /* + if($("#tour_video_holder").in_viewport()){ + if(self.tour_player.paused()){ + self.tour_player.play(); + } + } + else { + self.tour_player.pause(); + } + */ + } + } + + }, + + getBreakpointWid: function(){ + return parseInt($("#css_vp_wid").css("width"), 10); + } + +}; + + + + + diff --git a/marketing/resources/static/marketing/js/jquery-1.9.1.min.js b/marketing/resources/static/marketing/js/jquery-1.9.1.min.js new file mode 100644 index 0000000..32d50cb --- /dev/null +++ b/marketing/resources/static/marketing/js/jquery-1.9.1.min.js @@ -0,0 +1,5 @@ +/*! jQuery v1.9.1 | (c) 2005, 2012 jQuery Foundation, Inc. | jquery.org/license +//@ sourceMappingURL=jquery.min.map +*/(function(e,t){var n,r,i=typeof t,o=e.document,a=e.location,s=e.jQuery,u=e.$,l={},c=[],p="1.9.1",f=c.concat,d=c.push,h=c.slice,g=c.indexOf,m=l.toString,y=l.hasOwnProperty,v=p.trim,b=function(e,t){return new b.fn.init(e,t,r)},x=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,w=/\S+/g,T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,N=/^(?:(<[\w\W]+>)[^>]*|#([\w-]*))$/,C=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,k=/^[\],:{}\s]*$/,E=/(?:^|:|,)(?:\s*\[)+/g,S=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,A=/"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g,j=/^-ms-/,D=/-([\da-z])/gi,L=function(e,t){return t.toUpperCase()},H=function(e){(o.addEventListener||"load"===e.type||"complete"===o.readyState)&&(q(),b.ready())},q=function(){o.addEventListener?(o.removeEventListener("DOMContentLoaded",H,!1),e.removeEventListener("load",H,!1)):(o.detachEvent("onreadystatechange",H),e.detachEvent("onload",H))};b.fn=b.prototype={jquery:p,constructor:b,init:function(e,n,r){var i,a;if(!e)return this;if("string"==typeof e){if(i="<"===e.charAt(0)&&">"===e.charAt(e.length-1)&&e.length>=3?[null,e,null]:N.exec(e),!i||!i[1]&&n)return!n||n.jquery?(n||r).find(e):this.constructor(n).find(e);if(i[1]){if(n=n instanceof b?n[0]:n,b.merge(this,b.parseHTML(i[1],n&&n.nodeType?n.ownerDocument||n:o,!0)),C.test(i[1])&&b.isPlainObject(n))for(i in n)b.isFunction(this[i])?this[i](n[i]):this.attr(i,n[i]);return this}if(a=o.getElementById(i[2]),a&&a.parentNode){if(a.id!==i[2])return r.find(e);this.length=1,this[0]=a}return this.context=o,this.selector=e,this}return e.nodeType?(this.context=this[0]=e,this.length=1,this):b.isFunction(e)?r.ready(e):(e.selector!==t&&(this.selector=e.selector,this.context=e.context),b.makeArray(e,this))},selector:"",length:0,size:function(){return this.length},toArray:function(){return h.call(this)},get:function(e){return null==e?this.toArray():0>e?this[this.length+e]:this[e]},pushStack:function(e){var t=b.merge(this.constructor(),e);return t.prevObject=this,t.context=this.context,t},each:function(e,t){return b.each(this,e,t)},ready:function(e){return b.ready.promise().done(e),this},slice:function(){return this.pushStack(h.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(0>e?t:0);return this.pushStack(n>=0&&t>n?[this[n]]:[])},map:function(e){return this.pushStack(b.map(this,function(t,n){return e.call(t,n,t)}))},end:function(){return this.prevObject||this.constructor(null)},push:d,sort:[].sort,splice:[].splice},b.fn.init.prototype=b.fn,b.extend=b.fn.extend=function(){var e,n,r,i,o,a,s=arguments[0]||{},u=1,l=arguments.length,c=!1;for("boolean"==typeof s&&(c=s,s=arguments[1]||{},u=2),"object"==typeof s||b.isFunction(s)||(s={}),l===u&&(s=this,--u);l>u;u++)if(null!=(o=arguments[u]))for(i in o)e=s[i],r=o[i],s!==r&&(c&&r&&(b.isPlainObject(r)||(n=b.isArray(r)))?(n?(n=!1,a=e&&b.isArray(e)?e:[]):a=e&&b.isPlainObject(e)?e:{},s[i]=b.extend(c,a,r)):r!==t&&(s[i]=r));return s},b.extend({noConflict:function(t){return e.$===b&&(e.$=u),t&&e.jQuery===b&&(e.jQuery=s),b},isReady:!1,readyWait:1,holdReady:function(e){e?b.readyWait++:b.ready(!0)},ready:function(e){if(e===!0?!--b.readyWait:!b.isReady){if(!o.body)return setTimeout(b.ready);b.isReady=!0,e!==!0&&--b.readyWait>0||(n.resolveWith(o,[b]),b.fn.trigger&&b(o).trigger("ready").off("ready"))}},isFunction:function(e){return"function"===b.type(e)},isArray:Array.isArray||function(e){return"array"===b.type(e)},isWindow:function(e){return null!=e&&e==e.window},isNumeric:function(e){return!isNaN(parseFloat(e))&&isFinite(e)},type:function(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[m.call(e)]||"object":typeof e},isPlainObject:function(e){if(!e||"object"!==b.type(e)||e.nodeType||b.isWindow(e))return!1;try{if(e.constructor&&!y.call(e,"constructor")&&!y.call(e.constructor.prototype,"isPrototypeOf"))return!1}catch(n){return!1}var r;for(r in e);return r===t||y.call(e,r)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},error:function(e){throw Error(e)},parseHTML:function(e,t,n){if(!e||"string"!=typeof e)return null;"boolean"==typeof t&&(n=t,t=!1),t=t||o;var r=C.exec(e),i=!n&&[];return r?[t.createElement(r[1])]:(r=b.buildFragment([e],t,i),i&&b(i).remove(),b.merge([],r.childNodes))},parseJSON:function(n){return e.JSON&&e.JSON.parse?e.JSON.parse(n):null===n?n:"string"==typeof n&&(n=b.trim(n),n&&k.test(n.replace(S,"@").replace(A,"]").replace(E,"")))?Function("return "+n)():(b.error("Invalid JSON: "+n),t)},parseXML:function(n){var r,i;if(!n||"string"!=typeof n)return null;try{e.DOMParser?(i=new DOMParser,r=i.parseFromString(n,"text/xml")):(r=new ActiveXObject("Microsoft.XMLDOM"),r.async="false",r.loadXML(n))}catch(o){r=t}return r&&r.documentElement&&!r.getElementsByTagName("parsererror").length||b.error("Invalid XML: "+n),r},noop:function(){},globalEval:function(t){t&&b.trim(t)&&(e.execScript||function(t){e.eval.call(e,t)})(t)},camelCase:function(e){return e.replace(j,"ms-").replace(D,L)},nodeName:function(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()},each:function(e,t,n){var r,i=0,o=e.length,a=M(e);if(n){if(a){for(;o>i;i++)if(r=t.apply(e[i],n),r===!1)break}else for(i in e)if(r=t.apply(e[i],n),r===!1)break}else if(a){for(;o>i;i++)if(r=t.call(e[i],i,e[i]),r===!1)break}else for(i in e)if(r=t.call(e[i],i,e[i]),r===!1)break;return e},trim:v&&!v.call("\ufeff\u00a0")?function(e){return null==e?"":v.call(e)}:function(e){return null==e?"":(e+"").replace(T,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(M(Object(e))?b.merge(n,"string"==typeof e?[e]:e):d.call(n,e)),n},inArray:function(e,t,n){var r;if(t){if(g)return g.call(t,e,n);for(r=t.length,n=n?0>n?Math.max(0,r+n):n:0;r>n;n++)if(n in t&&t[n]===e)return n}return-1},merge:function(e,n){var r=n.length,i=e.length,o=0;if("number"==typeof r)for(;r>o;o++)e[i++]=n[o];else while(n[o]!==t)e[i++]=n[o++];return e.length=i,e},grep:function(e,t,n){var r,i=[],o=0,a=e.length;for(n=!!n;a>o;o++)r=!!t(e[o],o),n!==r&&i.push(e[o]);return i},map:function(e,t,n){var r,i=0,o=e.length,a=M(e),s=[];if(a)for(;o>i;i++)r=t(e[i],i,n),null!=r&&(s[s.length]=r);else for(i in e)r=t(e[i],i,n),null!=r&&(s[s.length]=r);return f.apply([],s)},guid:1,proxy:function(e,n){var r,i,o;return"string"==typeof n&&(o=e[n],n=e,e=o),b.isFunction(e)?(r=h.call(arguments,2),i=function(){return e.apply(n||this,r.concat(h.call(arguments)))},i.guid=e.guid=e.guid||b.guid++,i):t},access:function(e,n,r,i,o,a,s){var u=0,l=e.length,c=null==r;if("object"===b.type(r)){o=!0;for(u in r)b.access(e,n,u,r[u],!0,a,s)}else if(i!==t&&(o=!0,b.isFunction(i)||(s=!0),c&&(s?(n.call(e,i),n=null):(c=n,n=function(e,t,n){return c.call(b(e),n)})),n))for(;l>u;u++)n(e[u],r,s?i:i.call(e[u],u,n(e[u],r)));return o?e:c?n.call(e):l?n(e[0],r):a},now:function(){return(new Date).getTime()}}),b.ready.promise=function(t){if(!n)if(n=b.Deferred(),"complete"===o.readyState)setTimeout(b.ready);else if(o.addEventListener)o.addEventListener("DOMContentLoaded",H,!1),e.addEventListener("load",H,!1);else{o.attachEvent("onreadystatechange",H),e.attachEvent("onload",H);var r=!1;try{r=null==e.frameElement&&o.documentElement}catch(i){}r&&r.doScroll&&function a(){if(!b.isReady){try{r.doScroll("left")}catch(e){return setTimeout(a,50)}q(),b.ready()}}()}return n.promise(t)},b.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(e,t){l["[object "+t+"]"]=t.toLowerCase()});function M(e){var t=e.length,n=b.type(e);return b.isWindow(e)?!1:1===e.nodeType&&t?!0:"array"===n||"function"!==n&&(0===t||"number"==typeof t&&t>0&&t-1 in e)}r=b(o);var _={};function F(e){var t=_[e]={};return b.each(e.match(w)||[],function(e,n){t[n]=!0}),t}b.Callbacks=function(e){e="string"==typeof e?_[e]||F(e):b.extend({},e);var n,r,i,o,a,s,u=[],l=!e.once&&[],c=function(t){for(r=e.memory&&t,i=!0,a=s||0,s=0,o=u.length,n=!0;u&&o>a;a++)if(u[a].apply(t[0],t[1])===!1&&e.stopOnFalse){r=!1;break}n=!1,u&&(l?l.length&&c(l.shift()):r?u=[]:p.disable())},p={add:function(){if(u){var t=u.length;(function i(t){b.each(t,function(t,n){var r=b.type(n);"function"===r?e.unique&&p.has(n)||u.push(n):n&&n.length&&"string"!==r&&i(n)})})(arguments),n?o=u.length:r&&(s=t,c(r))}return this},remove:function(){return u&&b.each(arguments,function(e,t){var r;while((r=b.inArray(t,u,r))>-1)u.splice(r,1),n&&(o>=r&&o--,a>=r&&a--)}),this},has:function(e){return e?b.inArray(e,u)>-1:!(!u||!u.length)},empty:function(){return u=[],this},disable:function(){return u=l=r=t,this},disabled:function(){return!u},lock:function(){return l=t,r||p.disable(),this},locked:function(){return!l},fireWith:function(e,t){return t=t||[],t=[e,t.slice?t.slice():t],!u||i&&!l||(n?l.push(t):c(t)),this},fire:function(){return p.fireWith(this,arguments),this},fired:function(){return!!i}};return p},b.extend({Deferred:function(e){var t=[["resolve","done",b.Callbacks("once memory"),"resolved"],["reject","fail",b.Callbacks("once memory"),"rejected"],["notify","progress",b.Callbacks("memory")]],n="pending",r={state:function(){return n},always:function(){return i.done(arguments).fail(arguments),this},then:function(){var e=arguments;return b.Deferred(function(n){b.each(t,function(t,o){var a=o[0],s=b.isFunction(e[t])&&e[t];i[o[1]](function(){var e=s&&s.apply(this,arguments);e&&b.isFunction(e.promise)?e.promise().done(n.resolve).fail(n.reject).progress(n.notify):n[a+"With"](this===r?n.promise():this,s?[e]:arguments)})}),e=null}).promise()},promise:function(e){return null!=e?b.extend(e,r):r}},i={};return r.pipe=r.then,b.each(t,function(e,o){var a=o[2],s=o[3];r[o[1]]=a.add,s&&a.add(function(){n=s},t[1^e][2].disable,t[2][2].lock),i[o[0]]=function(){return i[o[0]+"With"](this===i?r:this,arguments),this},i[o[0]+"With"]=a.fireWith}),r.promise(i),e&&e.call(i,i),i},when:function(e){var t=0,n=h.call(arguments),r=n.length,i=1!==r||e&&b.isFunction(e.promise)?r:0,o=1===i?e:b.Deferred(),a=function(e,t,n){return function(r){t[e]=this,n[e]=arguments.length>1?h.call(arguments):r,n===s?o.notifyWith(t,n):--i||o.resolveWith(t,n)}},s,u,l;if(r>1)for(s=Array(r),u=Array(r),l=Array(r);r>t;t++)n[t]&&b.isFunction(n[t].promise)?n[t].promise().done(a(t,l,n)).fail(o.reject).progress(a(t,u,s)):--i;return i||o.resolveWith(l,n),o.promise()}}),b.support=function(){var t,n,r,a,s,u,l,c,p,f,d=o.createElement("div");if(d.setAttribute("className","t"),d.innerHTML="
a",n=d.getElementsByTagName("*"),r=d.getElementsByTagName("a")[0],!n||!r||!n.length)return{};s=o.createElement("select"),l=s.appendChild(o.createElement("option")),a=d.getElementsByTagName("input")[0],r.style.cssText="top:1px;float:left;opacity:.5",t={getSetAttribute:"t"!==d.className,leadingWhitespace:3===d.firstChild.nodeType,tbody:!d.getElementsByTagName("tbody").length,htmlSerialize:!!d.getElementsByTagName("link").length,style:/top/.test(r.getAttribute("style")),hrefNormalized:"/a"===r.getAttribute("href"),opacity:/^0.5/.test(r.style.opacity),cssFloat:!!r.style.cssFloat,checkOn:!!a.value,optSelected:l.selected,enctype:!!o.createElement("form").enctype,html5Clone:"<:nav>"!==o.createElement("nav").cloneNode(!0).outerHTML,boxModel:"CSS1Compat"===o.compatMode,deleteExpando:!0,noCloneEvent:!0,inlineBlockNeedsLayout:!1,shrinkWrapBlocks:!1,reliableMarginRight:!0,boxSizingReliable:!0,pixelPosition:!1},a.checked=!0,t.noCloneChecked=a.cloneNode(!0).checked,s.disabled=!0,t.optDisabled=!l.disabled;try{delete d.test}catch(h){t.deleteExpando=!1}a=o.createElement("input"),a.setAttribute("value",""),t.input=""===a.getAttribute("value"),a.value="t",a.setAttribute("type","radio"),t.radioValue="t"===a.value,a.setAttribute("checked","t"),a.setAttribute("name","t"),u=o.createDocumentFragment(),u.appendChild(a),t.appendChecked=a.checked,t.checkClone=u.cloneNode(!0).cloneNode(!0).lastChild.checked,d.attachEvent&&(d.attachEvent("onclick",function(){t.noCloneEvent=!1}),d.cloneNode(!0).click());for(f in{submit:!0,change:!0,focusin:!0})d.setAttribute(c="on"+f,"t"),t[f+"Bubbles"]=c in e||d.attributes[c].expando===!1;return d.style.backgroundClip="content-box",d.cloneNode(!0).style.backgroundClip="",t.clearCloneStyle="content-box"===d.style.backgroundClip,b(function(){var n,r,a,s="padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;",u=o.getElementsByTagName("body")[0];u&&(n=o.createElement("div"),n.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",u.appendChild(n).appendChild(d),d.innerHTML="
t
",a=d.getElementsByTagName("td"),a[0].style.cssText="padding:0;margin:0;border:0;display:none",p=0===a[0].offsetHeight,a[0].style.display="",a[1].style.display="none",t.reliableHiddenOffsets=p&&0===a[0].offsetHeight,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",t.boxSizing=4===d.offsetWidth,t.doesNotIncludeMarginInBodyOffset=1!==u.offsetTop,e.getComputedStyle&&(t.pixelPosition="1%"!==(e.getComputedStyle(d,null)||{}).top,t.boxSizingReliable="4px"===(e.getComputedStyle(d,null)||{width:"4px"}).width,r=d.appendChild(o.createElement("div")),r.style.cssText=d.style.cssText=s,r.style.marginRight=r.style.width="0",d.style.width="1px",t.reliableMarginRight=!parseFloat((e.getComputedStyle(r,null)||{}).marginRight)),typeof d.style.zoom!==i&&(d.innerHTML="",d.style.cssText=s+"width:1px;padding:1px;display:inline;zoom:1",t.inlineBlockNeedsLayout=3===d.offsetWidth,d.style.display="block",d.innerHTML="
",d.firstChild.style.width="5px",t.shrinkWrapBlocks=3!==d.offsetWidth,t.inlineBlockNeedsLayout&&(u.style.zoom=1)),u.removeChild(n),n=d=a=r=null)}),n=s=u=l=r=a=null,t}();var O=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,B=/([A-Z])/g;function P(e,n,r,i){if(b.acceptData(e)){var o,a,s=b.expando,u="string"==typeof n,l=e.nodeType,p=l?b.cache:e,f=l?e[s]:e[s]&&s;if(f&&p[f]&&(i||p[f].data)||!u||r!==t)return f||(l?e[s]=f=c.pop()||b.guid++:f=s),p[f]||(p[f]={},l||(p[f].toJSON=b.noop)),("object"==typeof n||"function"==typeof n)&&(i?p[f]=b.extend(p[f],n):p[f].data=b.extend(p[f].data,n)),o=p[f],i||(o.data||(o.data={}),o=o.data),r!==t&&(o[b.camelCase(n)]=r),u?(a=o[n],null==a&&(a=o[b.camelCase(n)])):a=o,a}}function R(e,t,n){if(b.acceptData(e)){var r,i,o,a=e.nodeType,s=a?b.cache:e,u=a?e[b.expando]:b.expando;if(s[u]){if(t&&(o=n?s[u]:s[u].data)){b.isArray(t)?t=t.concat(b.map(t,b.camelCase)):t in o?t=[t]:(t=b.camelCase(t),t=t in o?[t]:t.split(" "));for(r=0,i=t.length;i>r;r++)delete o[t[r]];if(!(n?$:b.isEmptyObject)(o))return}(n||(delete s[u].data,$(s[u])))&&(a?b.cleanData([e],!0):b.support.deleteExpando||s!=s.window?delete s[u]:s[u]=null)}}}b.extend({cache:{},expando:"jQuery"+(p+Math.random()).replace(/\D/g,""),noData:{embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:!0},hasData:function(e){return e=e.nodeType?b.cache[e[b.expando]]:e[b.expando],!!e&&!$(e)},data:function(e,t,n){return P(e,t,n)},removeData:function(e,t){return R(e,t)},_data:function(e,t,n){return P(e,t,n,!0)},_removeData:function(e,t){return R(e,t,!0)},acceptData:function(e){if(e.nodeType&&1!==e.nodeType&&9!==e.nodeType)return!1;var t=e.nodeName&&b.noData[e.nodeName.toLowerCase()];return!t||t!==!0&&e.getAttribute("classid")===t}}),b.fn.extend({data:function(e,n){var r,i,o=this[0],a=0,s=null;if(e===t){if(this.length&&(s=b.data(o),1===o.nodeType&&!b._data(o,"parsedAttrs"))){for(r=o.attributes;r.length>a;a++)i=r[a].name,i.indexOf("data-")||(i=b.camelCase(i.slice(5)),W(o,i,s[i]));b._data(o,"parsedAttrs",!0)}return s}return"object"==typeof e?this.each(function(){b.data(this,e)}):b.access(this,function(n){return n===t?o?W(o,e,b.data(o,e)):null:(this.each(function(){b.data(this,e,n)}),t)},null,n,arguments.length>1,null,!0)},removeData:function(e){return this.each(function(){b.removeData(this,e)})}});function W(e,n,r){if(r===t&&1===e.nodeType){var i="data-"+n.replace(B,"-$1").toLowerCase();if(r=e.getAttribute(i),"string"==typeof r){try{r="true"===r?!0:"false"===r?!1:"null"===r?null:+r+""===r?+r:O.test(r)?b.parseJSON(r):r}catch(o){}b.data(e,n,r)}else r=t}return r}function $(e){var t;for(t in e)if(("data"!==t||!b.isEmptyObject(e[t]))&&"toJSON"!==t)return!1;return!0}b.extend({queue:function(e,n,r){var i;return e?(n=(n||"fx")+"queue",i=b._data(e,n),r&&(!i||b.isArray(r)?i=b._data(e,n,b.makeArray(r)):i.push(r)),i||[]):t},dequeue:function(e,t){t=t||"fx";var n=b.queue(e,t),r=n.length,i=n.shift(),o=b._queueHooks(e,t),a=function(){b.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),o.cur=i,i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return b._data(e,n)||b._data(e,n,{empty:b.Callbacks("once memory").add(function(){b._removeData(e,t+"queue"),b._removeData(e,n)})})}}),b.fn.extend({queue:function(e,n){var r=2;return"string"!=typeof e&&(n=e,e="fx",r--),r>arguments.length?b.queue(this[0],e):n===t?this:this.each(function(){var t=b.queue(this,e,n);b._queueHooks(this,e),"fx"===e&&"inprogress"!==t[0]&&b.dequeue(this,e)})},dequeue:function(e){return this.each(function(){b.dequeue(this,e)})},delay:function(e,t){return e=b.fx?b.fx.speeds[e]||e:e,t=t||"fx",this.queue(t,function(t,n){var r=setTimeout(t,e);n.stop=function(){clearTimeout(r)}})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,n){var r,i=1,o=b.Deferred(),a=this,s=this.length,u=function(){--i||o.resolveWith(a,[a])};"string"!=typeof e&&(n=e,e=t),e=e||"fx";while(s--)r=b._data(a[s],e+"queueHooks"),r&&r.empty&&(i++,r.empty.add(u));return u(),o.promise(n)}});var I,z,X=/[\t\r\n]/g,U=/\r/g,V=/^(?:input|select|textarea|button|object)$/i,Y=/^(?:a|area)$/i,J=/^(?:checked|selected|autofocus|autoplay|async|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped)$/i,G=/^(?:checked|selected)$/i,Q=b.support.getSetAttribute,K=b.support.input;b.fn.extend({attr:function(e,t){return b.access(this,b.attr,e,t,arguments.length>1)},removeAttr:function(e){return this.each(function(){b.removeAttr(this,e)})},prop:function(e,t){return b.access(this,b.prop,e,t,arguments.length>1)},removeProp:function(e){return e=b.propFix[e]||e,this.each(function(){try{this[e]=t,delete this[e]}catch(n){}})},addClass:function(e){var t,n,r,i,o,a=0,s=this.length,u="string"==typeof e&&e;if(b.isFunction(e))return this.each(function(t){b(this).addClass(e.call(this,t,this.className))});if(u)for(t=(e||"").match(w)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(X," "):" ")){o=0;while(i=t[o++])0>r.indexOf(" "+i+" ")&&(r+=i+" ");n.className=b.trim(r)}return this},removeClass:function(e){var t,n,r,i,o,a=0,s=this.length,u=0===arguments.length||"string"==typeof e&&e;if(b.isFunction(e))return this.each(function(t){b(this).removeClass(e.call(this,t,this.className))});if(u)for(t=(e||"").match(w)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(X," "):"")){o=0;while(i=t[o++])while(r.indexOf(" "+i+" ")>=0)r=r.replace(" "+i+" "," ");n.className=e?b.trim(r):""}return this},toggleClass:function(e,t){var n=typeof e,r="boolean"==typeof t;return b.isFunction(e)?this.each(function(n){b(this).toggleClass(e.call(this,n,this.className,t),t)}):this.each(function(){if("string"===n){var o,a=0,s=b(this),u=t,l=e.match(w)||[];while(o=l[a++])u=r?u:!s.hasClass(o),s[u?"addClass":"removeClass"](o)}else(n===i||"boolean"===n)&&(this.className&&b._data(this,"__className__",this.className),this.className=this.className||e===!1?"":b._data(this,"__className__")||"")})},hasClass:function(e){var t=" "+e+" ",n=0,r=this.length;for(;r>n;n++)if(1===this[n].nodeType&&(" "+this[n].className+" ").replace(X," ").indexOf(t)>=0)return!0;return!1},val:function(e){var n,r,i,o=this[0];{if(arguments.length)return i=b.isFunction(e),this.each(function(n){var o,a=b(this);1===this.nodeType&&(o=i?e.call(this,n,a.val()):e,null==o?o="":"number"==typeof o?o+="":b.isArray(o)&&(o=b.map(o,function(e){return null==e?"":e+""})),r=b.valHooks[this.type]||b.valHooks[this.nodeName.toLowerCase()],r&&"set"in r&&r.set(this,o,"value")!==t||(this.value=o))});if(o)return r=b.valHooks[o.type]||b.valHooks[o.nodeName.toLowerCase()],r&&"get"in r&&(n=r.get(o,"value"))!==t?n:(n=o.value,"string"==typeof n?n.replace(U,""):null==n?"":n)}}}),b.extend({valHooks:{option:{get:function(e){var t=e.attributes.value;return!t||t.specified?e.value:e.text}},select:{get:function(e){var t,n,r=e.options,i=e.selectedIndex,o="select-one"===e.type||0>i,a=o?null:[],s=o?i+1:r.length,u=0>i?s:o?i:0;for(;s>u;u++)if(n=r[u],!(!n.selected&&u!==i||(b.support.optDisabled?n.disabled:null!==n.getAttribute("disabled"))||n.parentNode.disabled&&b.nodeName(n.parentNode,"optgroup"))){if(t=b(n).val(),o)return t;a.push(t)}return a},set:function(e,t){var n=b.makeArray(t);return b(e).find("option").each(function(){this.selected=b.inArray(b(this).val(),n)>=0}),n.length||(e.selectedIndex=-1),n}}},attr:function(e,n,r){var o,a,s,u=e.nodeType;if(e&&3!==u&&8!==u&&2!==u)return typeof e.getAttribute===i?b.prop(e,n,r):(a=1!==u||!b.isXMLDoc(e),a&&(n=n.toLowerCase(),o=b.attrHooks[n]||(J.test(n)?z:I)),r===t?o&&a&&"get"in o&&null!==(s=o.get(e,n))?s:(typeof e.getAttribute!==i&&(s=e.getAttribute(n)),null==s?t:s):null!==r?o&&a&&"set"in o&&(s=o.set(e,r,n))!==t?s:(e.setAttribute(n,r+""),r):(b.removeAttr(e,n),t))},removeAttr:function(e,t){var n,r,i=0,o=t&&t.match(w);if(o&&1===e.nodeType)while(n=o[i++])r=b.propFix[n]||n,J.test(n)?!Q&&G.test(n)?e[b.camelCase("default-"+n)]=e[r]=!1:e[r]=!1:b.attr(e,n,""),e.removeAttribute(Q?n:r)},attrHooks:{type:{set:function(e,t){if(!b.support.radioValue&&"radio"===t&&b.nodeName(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},propFix:{tabindex:"tabIndex",readonly:"readOnly","for":"htmlFor","class":"className",maxlength:"maxLength",cellspacing:"cellSpacing",cellpadding:"cellPadding",rowspan:"rowSpan",colspan:"colSpan",usemap:"useMap",frameborder:"frameBorder",contenteditable:"contentEditable"},prop:function(e,n,r){var i,o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return a=1!==s||!b.isXMLDoc(e),a&&(n=b.propFix[n]||n,o=b.propHooks[n]),r!==t?o&&"set"in o&&(i=o.set(e,r,n))!==t?i:e[n]=r:o&&"get"in o&&null!==(i=o.get(e,n))?i:e[n]},propHooks:{tabIndex:{get:function(e){var n=e.getAttributeNode("tabindex");return n&&n.specified?parseInt(n.value,10):V.test(e.nodeName)||Y.test(e.nodeName)&&e.href?0:t}}}}),z={get:function(e,n){var r=b.prop(e,n),i="boolean"==typeof r&&e.getAttribute(n),o="boolean"==typeof r?K&&Q?null!=i:G.test(n)?e[b.camelCase("default-"+n)]:!!i:e.getAttributeNode(n);return o&&o.value!==!1?n.toLowerCase():t},set:function(e,t,n){return t===!1?b.removeAttr(e,n):K&&Q||!G.test(n)?e.setAttribute(!Q&&b.propFix[n]||n,n):e[b.camelCase("default-"+n)]=e[n]=!0,n}},K&&Q||(b.attrHooks.value={get:function(e,n){var r=e.getAttributeNode(n);return b.nodeName(e,"input")?e.defaultValue:r&&r.specified?r.value:t},set:function(e,n,r){return b.nodeName(e,"input")?(e.defaultValue=n,t):I&&I.set(e,n,r)}}),Q||(I=b.valHooks.button={get:function(e,n){var r=e.getAttributeNode(n);return r&&("id"===n||"name"===n||"coords"===n?""!==r.value:r.specified)?r.value:t},set:function(e,n,r){var i=e.getAttributeNode(r);return i||e.setAttributeNode(i=e.ownerDocument.createAttribute(r)),i.value=n+="","value"===r||n===e.getAttribute(r)?n:t}},b.attrHooks.contenteditable={get:I.get,set:function(e,t,n){I.set(e,""===t?!1:t,n)}},b.each(["width","height"],function(e,n){b.attrHooks[n]=b.extend(b.attrHooks[n],{set:function(e,r){return""===r?(e.setAttribute(n,"auto"),r):t}})})),b.support.hrefNormalized||(b.each(["href","src","width","height"],function(e,n){b.attrHooks[n]=b.extend(b.attrHooks[n],{get:function(e){var r=e.getAttribute(n,2);return null==r?t:r}})}),b.each(["href","src"],function(e,t){b.propHooks[t]={get:function(e){return e.getAttribute(t,4)}}})),b.support.style||(b.attrHooks.style={get:function(e){return e.style.cssText||t},set:function(e,t){return e.style.cssText=t+""}}),b.support.optSelected||(b.propHooks.selected=b.extend(b.propHooks.selected,{get:function(e){var t=e.parentNode;return t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex),null}})),b.support.enctype||(b.propFix.enctype="encoding"),b.support.checkOn||b.each(["radio","checkbox"],function(){b.valHooks[this]={get:function(e){return null===e.getAttribute("value")?"on":e.value}}}),b.each(["radio","checkbox"],function(){b.valHooks[this]=b.extend(b.valHooks[this],{set:function(e,n){return b.isArray(n)?e.checked=b.inArray(b(e).val(),n)>=0:t}})});var Z=/^(?:input|select|textarea)$/i,et=/^key/,tt=/^(?:mouse|contextmenu)|click/,nt=/^(?:focusinfocus|focusoutblur)$/,rt=/^([^.]*)(?:\.(.+)|)$/;function it(){return!0}function ot(){return!1}b.event={global:{},add:function(e,n,r,o,a){var s,u,l,c,p,f,d,h,g,m,y,v=b._data(e);if(v){r.handler&&(c=r,r=c.handler,a=c.selector),r.guid||(r.guid=b.guid++),(u=v.events)||(u=v.events={}),(f=v.handle)||(f=v.handle=function(e){return typeof b===i||e&&b.event.triggered===e.type?t:b.event.dispatch.apply(f.elem,arguments)},f.elem=e),n=(n||"").match(w)||[""],l=n.length;while(l--)s=rt.exec(n[l])||[],g=y=s[1],m=(s[2]||"").split(".").sort(),p=b.event.special[g]||{},g=(a?p.delegateType:p.bindType)||g,p=b.event.special[g]||{},d=b.extend({type:g,origType:y,data:o,handler:r,guid:r.guid,selector:a,needsContext:a&&b.expr.match.needsContext.test(a),namespace:m.join(".")},c),(h=u[g])||(h=u[g]=[],h.delegateCount=0,p.setup&&p.setup.call(e,o,m,f)!==!1||(e.addEventListener?e.addEventListener(g,f,!1):e.attachEvent&&e.attachEvent("on"+g,f))),p.add&&(p.add.call(e,d),d.handler.guid||(d.handler.guid=r.guid)),a?h.splice(h.delegateCount++,0,d):h.push(d),b.event.global[g]=!0;e=null}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,p,f,d,h,g,m=b.hasData(e)&&b._data(e);if(m&&(c=m.events)){t=(t||"").match(w)||[""],l=t.length;while(l--)if(s=rt.exec(t[l])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){p=b.event.special[d]||{},d=(r?p.delegateType:p.bindType)||d,f=c[d]||[],s=s[2]&&RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),u=o=f.length;while(o--)a=f[o],!i&&g!==a.origType||n&&n.guid!==a.guid||s&&!s.test(a.namespace)||r&&r!==a.selector&&("**"!==r||!a.selector)||(f.splice(o,1),a.selector&&f.delegateCount--,p.remove&&p.remove.call(e,a));u&&!f.length&&(p.teardown&&p.teardown.call(e,h,m.handle)!==!1||b.removeEvent(e,d,m.handle),delete c[d])}else for(d in c)b.event.remove(e,d+t[l],n,r,!0);b.isEmptyObject(c)&&(delete m.handle,b._removeData(e,"events"))}},trigger:function(n,r,i,a){var s,u,l,c,p,f,d,h=[i||o],g=y.call(n,"type")?n.type:n,m=y.call(n,"namespace")?n.namespace.split("."):[];if(l=f=i=i||o,3!==i.nodeType&&8!==i.nodeType&&!nt.test(g+b.event.triggered)&&(g.indexOf(".")>=0&&(m=g.split("."),g=m.shift(),m.sort()),u=0>g.indexOf(":")&&"on"+g,n=n[b.expando]?n:new b.Event(g,"object"==typeof n&&n),n.isTrigger=!0,n.namespace=m.join("."),n.namespace_re=n.namespace?RegExp("(^|\\.)"+m.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,n.result=t,n.target||(n.target=i),r=null==r?[n]:b.makeArray(r,[n]),p=b.event.special[g]||{},a||!p.trigger||p.trigger.apply(i,r)!==!1)){if(!a&&!p.noBubble&&!b.isWindow(i)){for(c=p.delegateType||g,nt.test(c+g)||(l=l.parentNode);l;l=l.parentNode)h.push(l),f=l;f===(i.ownerDocument||o)&&h.push(f.defaultView||f.parentWindow||e)}d=0;while((l=h[d++])&&!n.isPropagationStopped())n.type=d>1?c:p.bindType||g,s=(b._data(l,"events")||{})[n.type]&&b._data(l,"handle"),s&&s.apply(l,r),s=u&&l[u],s&&b.acceptData(l)&&s.apply&&s.apply(l,r)===!1&&n.preventDefault();if(n.type=g,!(a||n.isDefaultPrevented()||p._default&&p._default.apply(i.ownerDocument,r)!==!1||"click"===g&&b.nodeName(i,"a")||!b.acceptData(i)||!u||!i[g]||b.isWindow(i))){f=i[u],f&&(i[u]=null),b.event.triggered=g;try{i[g]()}catch(v){}b.event.triggered=t,f&&(i[u]=f)}return n.result}},dispatch:function(e){e=b.event.fix(e);var n,r,i,o,a,s=[],u=h.call(arguments),l=(b._data(this,"events")||{})[e.type]||[],c=b.event.special[e.type]||{};if(u[0]=e,e.delegateTarget=this,!c.preDispatch||c.preDispatch.call(this,e)!==!1){s=b.event.handlers.call(this,e,l),n=0;while((o=s[n++])&&!e.isPropagationStopped()){e.currentTarget=o.elem,a=0;while((i=o.handlers[a++])&&!e.isImmediatePropagationStopped())(!e.namespace_re||e.namespace_re.test(i.namespace))&&(e.handleObj=i,e.data=i.data,r=((b.event.special[i.origType]||{}).handle||i.handler).apply(o.elem,u),r!==t&&(e.result=r)===!1&&(e.preventDefault(),e.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,e),e.result}},handlers:function(e,n){var r,i,o,a,s=[],u=n.delegateCount,l=e.target;if(u&&l.nodeType&&(!e.button||"click"!==e.type))for(;l!=this;l=l.parentNode||this)if(1===l.nodeType&&(l.disabled!==!0||"click"!==e.type)){for(o=[],a=0;u>a;a++)i=n[a],r=i.selector+" ",o[r]===t&&(o[r]=i.needsContext?b(r,this).index(l)>=0:b.find(r,this,null,[l]).length),o[r]&&o.push(i);o.length&&s.push({elem:l,handlers:o})}return n.length>u&&s.push({elem:this,handlers:n.slice(u)}),s},fix:function(e){if(e[b.expando])return e;var t,n,r,i=e.type,a=e,s=this.fixHooks[i];s||(this.fixHooks[i]=s=tt.test(i)?this.mouseHooks:et.test(i)?this.keyHooks:{}),r=s.props?this.props.concat(s.props):this.props,e=new b.Event(a),t=r.length;while(t--)n=r[t],e[n]=a[n];return e.target||(e.target=a.srcElement||o),3===e.target.nodeType&&(e.target=e.target.parentNode),e.metaKey=!!e.metaKey,s.filter?s.filter(e,a):e},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(e,t){return null==e.which&&(e.which=null!=t.charCode?t.charCode:t.keyCode),e}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(e,n){var r,i,a,s=n.button,u=n.fromElement;return null==e.pageX&&null!=n.clientX&&(i=e.target.ownerDocument||o,a=i.documentElement,r=i.body,e.pageX=n.clientX+(a&&a.scrollLeft||r&&r.scrollLeft||0)-(a&&a.clientLeft||r&&r.clientLeft||0),e.pageY=n.clientY+(a&&a.scrollTop||r&&r.scrollTop||0)-(a&&a.clientTop||r&&r.clientTop||0)),!e.relatedTarget&&u&&(e.relatedTarget=u===e.target?n.toElement:u),e.which||s===t||(e.which=1&s?1:2&s?3:4&s?2:0),e}},special:{load:{noBubble:!0},click:{trigger:function(){return b.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):t}},focus:{trigger:function(){if(this!==o.activeElement&&this.focus)try{return this.focus(),!1}catch(e){}},delegateType:"focusin"},blur:{trigger:function(){return this===o.activeElement&&this.blur?(this.blur(),!1):t},delegateType:"focusout"},beforeunload:{postDispatch:function(e){e.result!==t&&(e.originalEvent.returnValue=e.result)}}},simulate:function(e,t,n,r){var i=b.extend(new b.Event,n,{type:e,isSimulated:!0,originalEvent:{}});r?b.event.trigger(i,null,t):b.event.dispatch.call(t,i),i.isDefaultPrevented()&&n.preventDefault()}},b.removeEvent=o.removeEventListener?function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n,!1)}:function(e,t,n){var r="on"+t;e.detachEvent&&(typeof e[r]===i&&(e[r]=null),e.detachEvent(r,n))},b.Event=function(e,n){return this instanceof b.Event?(e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||e.returnValue===!1||e.getPreventDefault&&e.getPreventDefault()?it:ot):this.type=e,n&&b.extend(this,n),this.timeStamp=e&&e.timeStamp||b.now(),this[b.expando]=!0,t):new b.Event(e,n)},b.Event.prototype={isDefaultPrevented:ot,isPropagationStopped:ot,isImmediatePropagationStopped:ot,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=it,e&&(e.preventDefault?e.preventDefault():e.returnValue=!1)},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=it,e&&(e.stopPropagation&&e.stopPropagation(),e.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=it,this.stopPropagation()}},b.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(e,t){b.event.special[e]={delegateType:t,bindType:t,handle:function(e){var n,r=this,i=e.relatedTarget,o=e.handleObj; +return(!i||i!==r&&!b.contains(r,i))&&(e.type=o.origType,n=o.handler.apply(this,arguments),e.type=t),n}}}),b.support.submitBubbles||(b.event.special.submit={setup:function(){return b.nodeName(this,"form")?!1:(b.event.add(this,"click._submit keypress._submit",function(e){var n=e.target,r=b.nodeName(n,"input")||b.nodeName(n,"button")?n.form:t;r&&!b._data(r,"submitBubbles")&&(b.event.add(r,"submit._submit",function(e){e._submit_bubble=!0}),b._data(r,"submitBubbles",!0))}),t)},postDispatch:function(e){e._submit_bubble&&(delete e._submit_bubble,this.parentNode&&!e.isTrigger&&b.event.simulate("submit",this.parentNode,e,!0))},teardown:function(){return b.nodeName(this,"form")?!1:(b.event.remove(this,"._submit"),t)}}),b.support.changeBubbles||(b.event.special.change={setup:function(){return Z.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(b.event.add(this,"propertychange._change",function(e){"checked"===e.originalEvent.propertyName&&(this._just_changed=!0)}),b.event.add(this,"click._change",function(e){this._just_changed&&!e.isTrigger&&(this._just_changed=!1),b.event.simulate("change",this,e,!0)})),!1):(b.event.add(this,"beforeactivate._change",function(e){var t=e.target;Z.test(t.nodeName)&&!b._data(t,"changeBubbles")&&(b.event.add(t,"change._change",function(e){!this.parentNode||e.isSimulated||e.isTrigger||b.event.simulate("change",this.parentNode,e,!0)}),b._data(t,"changeBubbles",!0))}),t)},handle:function(e){var n=e.target;return this!==n||e.isSimulated||e.isTrigger||"radio"!==n.type&&"checkbox"!==n.type?e.handleObj.handler.apply(this,arguments):t},teardown:function(){return b.event.remove(this,"._change"),!Z.test(this.nodeName)}}),b.support.focusinBubbles||b.each({focus:"focusin",blur:"focusout"},function(e,t){var n=0,r=function(e){b.event.simulate(t,e.target,b.event.fix(e),!0)};b.event.special[t]={setup:function(){0===n++&&o.addEventListener(e,r,!0)},teardown:function(){0===--n&&o.removeEventListener(e,r,!0)}}}),b.fn.extend({on:function(e,n,r,i,o){var a,s;if("object"==typeof e){"string"!=typeof n&&(r=r||n,n=t);for(a in e)this.on(a,n,r,e[a],o);return this}if(null==r&&null==i?(i=n,r=n=t):null==i&&("string"==typeof n?(i=r,r=t):(i=r,r=n,n=t)),i===!1)i=ot;else if(!i)return this;return 1===o&&(s=i,i=function(e){return b().off(e),s.apply(this,arguments)},i.guid=s.guid||(s.guid=b.guid++)),this.each(function(){b.event.add(this,e,i,r,n)})},one:function(e,t,n,r){return this.on(e,t,n,r,1)},off:function(e,n,r){var i,o;if(e&&e.preventDefault&&e.handleObj)return i=e.handleObj,b(e.delegateTarget).off(i.namespace?i.origType+"."+i.namespace:i.origType,i.selector,i.handler),this;if("object"==typeof e){for(o in e)this.off(o,n,e[o]);return this}return(n===!1||"function"==typeof n)&&(r=n,n=t),r===!1&&(r=ot),this.each(function(){b.event.remove(this,e,r,n)})},bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},trigger:function(e,t){return this.each(function(){b.event.trigger(e,t,this)})},triggerHandler:function(e,n){var r=this[0];return r?b.event.trigger(e,n,r,!0):t}}),function(e,t){var n,r,i,o,a,s,u,l,c,p,f,d,h,g,m,y,v,x="sizzle"+-new Date,w=e.document,T={},N=0,C=0,k=it(),E=it(),S=it(),A=typeof t,j=1<<31,D=[],L=D.pop,H=D.push,q=D.slice,M=D.indexOf||function(e){var t=0,n=this.length;for(;n>t;t++)if(this[t]===e)return t;return-1},_="[\\x20\\t\\r\\n\\f]",F="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",O=F.replace("w","w#"),B="([*^$|!~]?=)",P="\\["+_+"*("+F+")"+_+"*(?:"+B+_+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+O+")|)|)"+_+"*\\]",R=":("+F+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+P.replace(3,8)+")*)|.*)\\)|)",W=RegExp("^"+_+"+|((?:^|[^\\\\])(?:\\\\.)*)"+_+"+$","g"),$=RegExp("^"+_+"*,"+_+"*"),I=RegExp("^"+_+"*([\\x20\\t\\r\\n\\f>+~])"+_+"*"),z=RegExp(R),X=RegExp("^"+O+"$"),U={ID:RegExp("^#("+F+")"),CLASS:RegExp("^\\.("+F+")"),NAME:RegExp("^\\[name=['\"]?("+F+")['\"]?\\]"),TAG:RegExp("^("+F.replace("w","w*")+")"),ATTR:RegExp("^"+P),PSEUDO:RegExp("^"+R),CHILD:RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+_+"*(even|odd|(([+-]|)(\\d*)n|)"+_+"*(?:([+-]|)"+_+"*(\\d+)|))"+_+"*\\)|)","i"),needsContext:RegExp("^"+_+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+_+"*((?:-\\d)?\\d*)"+_+"*\\)|)(?=[^-]|$)","i")},V=/[\x20\t\r\n\f]*[+~]/,Y=/^[^{]+\{\s*\[native code/,J=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,G=/^(?:input|select|textarea|button)$/i,Q=/^h\d$/i,K=/'|\\/g,Z=/\=[\x20\t\r\n\f]*([^'"\]]*)[\x20\t\r\n\f]*\]/g,et=/\\([\da-fA-F]{1,6}[\x20\t\r\n\f]?|.)/g,tt=function(e,t){var n="0x"+t-65536;return n!==n?t:0>n?String.fromCharCode(n+65536):String.fromCharCode(55296|n>>10,56320|1023&n)};try{q.call(w.documentElement.childNodes,0)[0].nodeType}catch(nt){q=function(e){var t,n=[];while(t=this[e++])n.push(t);return n}}function rt(e){return Y.test(e+"")}function it(){var e,t=[];return e=function(n,r){return t.push(n+=" ")>i.cacheLength&&delete e[t.shift()],e[n]=r}}function ot(e){return e[x]=!0,e}function at(e){var t=p.createElement("div");try{return e(t)}catch(n){return!1}finally{t=null}}function st(e,t,n,r){var i,o,a,s,u,l,f,g,m,v;if((t?t.ownerDocument||t:w)!==p&&c(t),t=t||p,n=n||[],!e||"string"!=typeof e)return n;if(1!==(s=t.nodeType)&&9!==s)return[];if(!d&&!r){if(i=J.exec(e))if(a=i[1]){if(9===s){if(o=t.getElementById(a),!o||!o.parentNode)return n;if(o.id===a)return n.push(o),n}else if(t.ownerDocument&&(o=t.ownerDocument.getElementById(a))&&y(t,o)&&o.id===a)return n.push(o),n}else{if(i[2])return H.apply(n,q.call(t.getElementsByTagName(e),0)),n;if((a=i[3])&&T.getByClassName&&t.getElementsByClassName)return H.apply(n,q.call(t.getElementsByClassName(a),0)),n}if(T.qsa&&!h.test(e)){if(f=!0,g=x,m=t,v=9===s&&e,1===s&&"object"!==t.nodeName.toLowerCase()){l=ft(e),(f=t.getAttribute("id"))?g=f.replace(K,"\\$&"):t.setAttribute("id",g),g="[id='"+g+"'] ",u=l.length;while(u--)l[u]=g+dt(l[u]);m=V.test(e)&&t.parentNode||t,v=l.join(",")}if(v)try{return H.apply(n,q.call(m.querySelectorAll(v),0)),n}catch(b){}finally{f||t.removeAttribute("id")}}}return wt(e.replace(W,"$1"),t,n,r)}a=st.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return t?"HTML"!==t.nodeName:!1},c=st.setDocument=function(e){var n=e?e.ownerDocument||e:w;return n!==p&&9===n.nodeType&&n.documentElement?(p=n,f=n.documentElement,d=a(n),T.tagNameNoComments=at(function(e){return e.appendChild(n.createComment("")),!e.getElementsByTagName("*").length}),T.attributes=at(function(e){e.innerHTML="";var t=typeof e.lastChild.getAttribute("multiple");return"boolean"!==t&&"string"!==t}),T.getByClassName=at(function(e){return e.innerHTML="",e.getElementsByClassName&&e.getElementsByClassName("e").length?(e.lastChild.className="e",2===e.getElementsByClassName("e").length):!1}),T.getByName=at(function(e){e.id=x+0,e.innerHTML="
",f.insertBefore(e,f.firstChild);var t=n.getElementsByName&&n.getElementsByName(x).length===2+n.getElementsByName(x+0).length;return T.getIdNotName=!n.getElementById(x),f.removeChild(e),t}),i.attrHandle=at(function(e){return e.innerHTML="",e.firstChild&&typeof e.firstChild.getAttribute!==A&&"#"===e.firstChild.getAttribute("href")})?{}:{href:function(e){return e.getAttribute("href",2)},type:function(e){return e.getAttribute("type")}},T.getIdNotName?(i.find.ID=function(e,t){if(typeof t.getElementById!==A&&!d){var n=t.getElementById(e);return n&&n.parentNode?[n]:[]}},i.filter.ID=function(e){var t=e.replace(et,tt);return function(e){return e.getAttribute("id")===t}}):(i.find.ID=function(e,n){if(typeof n.getElementById!==A&&!d){var r=n.getElementById(e);return r?r.id===e||typeof r.getAttributeNode!==A&&r.getAttributeNode("id").value===e?[r]:t:[]}},i.filter.ID=function(e){var t=e.replace(et,tt);return function(e){var n=typeof e.getAttributeNode!==A&&e.getAttributeNode("id");return n&&n.value===t}}),i.find.TAG=T.tagNameNoComments?function(e,n){return typeof n.getElementsByTagName!==A?n.getElementsByTagName(e):t}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},i.find.NAME=T.getByName&&function(e,n){return typeof n.getElementsByName!==A?n.getElementsByName(name):t},i.find.CLASS=T.getByClassName&&function(e,n){return typeof n.getElementsByClassName===A||d?t:n.getElementsByClassName(e)},g=[],h=[":focus"],(T.qsa=rt(n.querySelectorAll))&&(at(function(e){e.innerHTML="",e.querySelectorAll("[selected]").length||h.push("\\["+_+"*(?:checked|disabled|ismap|multiple|readonly|selected|value)"),e.querySelectorAll(":checked").length||h.push(":checked")}),at(function(e){e.innerHTML="",e.querySelectorAll("[i^='']").length&&h.push("[*^$]="+_+"*(?:\"\"|'')"),e.querySelectorAll(":enabled").length||h.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),h.push(",.*:")})),(T.matchesSelector=rt(m=f.matchesSelector||f.mozMatchesSelector||f.webkitMatchesSelector||f.oMatchesSelector||f.msMatchesSelector))&&at(function(e){T.disconnectedMatch=m.call(e,"div"),m.call(e,"[s!='']:x"),g.push("!=",R)}),h=RegExp(h.join("|")),g=RegExp(g.join("|")),y=rt(f.contains)||f.compareDocumentPosition?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},v=f.compareDocumentPosition?function(e,t){var r;return e===t?(u=!0,0):(r=t.compareDocumentPosition&&e.compareDocumentPosition&&e.compareDocumentPosition(t))?1&r||e.parentNode&&11===e.parentNode.nodeType?e===n||y(w,e)?-1:t===n||y(w,t)?1:0:4&r?-1:1:e.compareDocumentPosition?-1:1}:function(e,t){var r,i=0,o=e.parentNode,a=t.parentNode,s=[e],l=[t];if(e===t)return u=!0,0;if(!o||!a)return e===n?-1:t===n?1:o?-1:a?1:0;if(o===a)return ut(e,t);r=e;while(r=r.parentNode)s.unshift(r);r=t;while(r=r.parentNode)l.unshift(r);while(s[i]===l[i])i++;return i?ut(s[i],l[i]):s[i]===w?-1:l[i]===w?1:0},u=!1,[0,0].sort(v),T.detectDuplicates=u,p):p},st.matches=function(e,t){return st(e,null,null,t)},st.matchesSelector=function(e,t){if((e.ownerDocument||e)!==p&&c(e),t=t.replace(Z,"='$1']"),!(!T.matchesSelector||d||g&&g.test(t)||h.test(t)))try{var n=m.call(e,t);if(n||T.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(r){}return st(t,p,null,[e]).length>0},st.contains=function(e,t){return(e.ownerDocument||e)!==p&&c(e),y(e,t)},st.attr=function(e,t){var n;return(e.ownerDocument||e)!==p&&c(e),d||(t=t.toLowerCase()),(n=i.attrHandle[t])?n(e):d||T.attributes?e.getAttribute(t):((n=e.getAttributeNode(t))||e.getAttribute(t))&&e[t]===!0?t:n&&n.specified?n.value:null},st.error=function(e){throw Error("Syntax error, unrecognized expression: "+e)},st.uniqueSort=function(e){var t,n=[],r=1,i=0;if(u=!T.detectDuplicates,e.sort(v),u){for(;t=e[r];r++)t===e[r-1]&&(i=n.push(r));while(i--)e.splice(n[i],1)}return e};function ut(e,t){var n=t&&e,r=n&&(~t.sourceIndex||j)-(~e.sourceIndex||j);if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function lt(e){return function(t){var n=t.nodeName.toLowerCase();return"input"===n&&t.type===e}}function ct(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function pt(e){return ot(function(t){return t=+t,ot(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}o=st.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else for(;t=e[r];r++)n+=o(t);return n},i=st.selectors={cacheLength:50,createPseudo:ot,match:U,find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(et,tt),e[3]=(e[4]||e[5]||"").replace(et,tt),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||st.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&st.error(e[0]),e},PSEUDO:function(e){var t,n=!e[5]&&e[2];return U.CHILD.test(e[0])?null:(e[4]?e[2]=e[4]:n&&z.test(n)&&(t=ft(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){return"*"===e?function(){return!0}:(e=e.replace(et,tt).toLowerCase(),function(t){return t.nodeName&&t.nodeName.toLowerCase()===e})},CLASS:function(e){var t=k[e+" "];return t||(t=RegExp("(^|"+_+")"+e+"("+_+"|$)"))&&k(e,function(e){return t.test(e.className||typeof e.getAttribute!==A&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=st.attr(r,e);return null==i?"!="===t:t?(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i+" ").indexOf(n)>-1:"|="===t?i===n||i.slice(0,n.length+1)===n+"-":!1):!0}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,u){var l,c,p,f,d,h,g=o!==a?"nextSibling":"previousSibling",m=t.parentNode,y=s&&t.nodeName.toLowerCase(),v=!u&&!s;if(m){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?m.firstChild:m.lastChild],a&&v){c=m[x]||(m[x]={}),l=c[e]||[],d=l[0]===N&&l[1],f=l[0]===N&&l[2],p=d&&m.childNodes[d];while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if(1===p.nodeType&&++f&&p===t){c[e]=[N,d,f];break}}else if(v&&(l=(t[x]||(t[x]={}))[e])&&l[0]===N)f=l[1];else while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===y:1===p.nodeType)&&++f&&(v&&((p[x]||(p[x]={}))[e]=[N,f]),p===t))break;return f-=i,f===r||0===f%r&&f/r>=0}}},PSEUDO:function(e,t){var n,r=i.pseudos[e]||i.setFilters[e.toLowerCase()]||st.error("unsupported pseudo: "+e);return r[x]?r(t):r.length>1?(n=[e,e,"",t],i.setFilters.hasOwnProperty(e.toLowerCase())?ot(function(e,n){var i,o=r(e,t),a=o.length;while(a--)i=M.call(e,o[a]),e[i]=!(n[i]=o[a])}):function(e){return r(e,0,n)}):r}},pseudos:{not:ot(function(e){var t=[],n=[],r=s(e.replace(W,"$1"));return r[x]?ot(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),!n.pop()}}),has:ot(function(e){return function(t){return st(e,t).length>0}}),contains:ot(function(e){return function(t){return(t.textContent||t.innerText||o(t)).indexOf(e)>-1}}),lang:ot(function(e){return X.test(e||"")||st.error("unsupported lang: "+e),e=e.replace(et,tt).toLowerCase(),function(t){var n;do if(n=d?t.getAttribute("xml:lang")||t.getAttribute("lang"):t.lang)return n=n.toLowerCase(),n===e||0===n.indexOf(e+"-");while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===f},focus:function(e){return e===p.activeElement&&(!p.hasFocus||p.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:function(e){return e.disabled===!1},disabled:function(e){return e.disabled===!0},checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,e.selected===!0},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeName>"@"||3===e.nodeType||4===e.nodeType)return!1;return!0},parent:function(e){return!i.pseudos.empty(e)},header:function(e){return Q.test(e.nodeName)},input:function(e){return G.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||t.toLowerCase()===e.type)},first:pt(function(){return[0]}),last:pt(function(e,t){return[t-1]}),eq:pt(function(e,t,n){return[0>n?n+t:n]}),even:pt(function(e,t){var n=0;for(;t>n;n+=2)e.push(n);return e}),odd:pt(function(e,t){var n=1;for(;t>n;n+=2)e.push(n);return e}),lt:pt(function(e,t,n){var r=0>n?n+t:n;for(;--r>=0;)e.push(r);return e}),gt:pt(function(e,t,n){var r=0>n?n+t:n;for(;t>++r;)e.push(r);return e})}};for(n in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})i.pseudos[n]=lt(n);for(n in{submit:!0,reset:!0})i.pseudos[n]=ct(n);function ft(e,t){var n,r,o,a,s,u,l,c=E[e+" "];if(c)return t?0:c.slice(0);s=e,u=[],l=i.preFilter;while(s){(!n||(r=$.exec(s)))&&(r&&(s=s.slice(r[0].length)||s),u.push(o=[])),n=!1,(r=I.exec(s))&&(n=r.shift(),o.push({value:n,type:r[0].replace(W," ")}),s=s.slice(n.length));for(a in i.filter)!(r=U[a].exec(s))||l[a]&&!(r=l[a](r))||(n=r.shift(),o.push({value:n,type:a,matches:r}),s=s.slice(n.length));if(!n)break}return t?s.length:s?st.error(e):E(e,u).slice(0)}function dt(e){var t=0,n=e.length,r="";for(;n>t;t++)r+=e[t].value;return r}function ht(e,t,n){var i=t.dir,o=n&&"parentNode"===i,a=C++;return t.first?function(t,n,r){while(t=t[i])if(1===t.nodeType||o)return e(t,n,r)}:function(t,n,s){var u,l,c,p=N+" "+a;if(s){while(t=t[i])if((1===t.nodeType||o)&&e(t,n,s))return!0}else while(t=t[i])if(1===t.nodeType||o)if(c=t[x]||(t[x]={}),(l=c[i])&&l[0]===p){if((u=l[1])===!0||u===r)return u===!0}else if(l=c[i]=[p],l[1]=e(t,n,s)||r,l[1]===!0)return!0}}function gt(e){return e.length>1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function mt(e,t,n,r,i){var o,a=[],s=0,u=e.length,l=null!=t;for(;u>s;s++)(o=e[s])&&(!n||n(o,r,i))&&(a.push(o),l&&t.push(s));return a}function yt(e,t,n,r,i,o){return r&&!r[x]&&(r=yt(r)),i&&!i[x]&&(i=yt(i,o)),ot(function(o,a,s,u){var l,c,p,f=[],d=[],h=a.length,g=o||xt(t||"*",s.nodeType?[s]:s,[]),m=!e||!o&&t?g:mt(g,f,e,s,u),y=n?i||(o?e:h||r)?[]:a:m;if(n&&n(m,y,s,u),r){l=mt(y,d),r(l,[],s,u),c=l.length;while(c--)(p=l[c])&&(y[d[c]]=!(m[d[c]]=p))}if(o){if(i||e){if(i){l=[],c=y.length;while(c--)(p=y[c])&&l.push(m[c]=p);i(null,y=[],l,u)}c=y.length;while(c--)(p=y[c])&&(l=i?M.call(o,p):f[c])>-1&&(o[l]=!(a[l]=p))}}else y=mt(y===a?y.splice(h,y.length):y),i?i(null,a,y,u):H.apply(a,y)})}function vt(e){var t,n,r,o=e.length,a=i.relative[e[0].type],s=a||i.relative[" "],u=a?1:0,c=ht(function(e){return e===t},s,!0),p=ht(function(e){return M.call(t,e)>-1},s,!0),f=[function(e,n,r){return!a&&(r||n!==l)||((t=n).nodeType?c(e,n,r):p(e,n,r))}];for(;o>u;u++)if(n=i.relative[e[u].type])f=[ht(gt(f),n)];else{if(n=i.filter[e[u].type].apply(null,e[u].matches),n[x]){for(r=++u;o>r;r++)if(i.relative[e[r].type])break;return yt(u>1&>(f),u>1&&dt(e.slice(0,u-1)).replace(W,"$1"),n,r>u&&vt(e.slice(u,r)),o>r&&vt(e=e.slice(r)),o>r&&dt(e))}f.push(n)}return gt(f)}function bt(e,t){var n=0,o=t.length>0,a=e.length>0,s=function(s,u,c,f,d){var h,g,m,y=[],v=0,b="0",x=s&&[],w=null!=d,T=l,C=s||a&&i.find.TAG("*",d&&u.parentNode||u),k=N+=null==T?1:Math.random()||.1;for(w&&(l=u!==p&&u,r=n);null!=(h=C[b]);b++){if(a&&h){g=0;while(m=e[g++])if(m(h,u,c)){f.push(h);break}w&&(N=k,r=++n)}o&&((h=!m&&h)&&v--,s&&x.push(h))}if(v+=b,o&&b!==v){g=0;while(m=t[g++])m(x,y,u,c);if(s){if(v>0)while(b--)x[b]||y[b]||(y[b]=L.call(f));y=mt(y)}H.apply(f,y),w&&!s&&y.length>0&&v+t.length>1&&st.uniqueSort(f)}return w&&(N=k,l=T),x};return o?ot(s):s}s=st.compile=function(e,t){var n,r=[],i=[],o=S[e+" "];if(!o){t||(t=ft(e)),n=t.length;while(n--)o=vt(t[n]),o[x]?r.push(o):i.push(o);o=S(e,bt(i,r))}return o};function xt(e,t,n){var r=0,i=t.length;for(;i>r;r++)st(e,t[r],n);return n}function wt(e,t,n,r){var o,a,u,l,c,p=ft(e);if(!r&&1===p.length){if(a=p[0]=p[0].slice(0),a.length>2&&"ID"===(u=a[0]).type&&9===t.nodeType&&!d&&i.relative[a[1].type]){if(t=i.find.ID(u.matches[0].replace(et,tt),t)[0],!t)return n;e=e.slice(a.shift().value.length)}o=U.needsContext.test(e)?0:a.length;while(o--){if(u=a[o],i.relative[l=u.type])break;if((c=i.find[l])&&(r=c(u.matches[0].replace(et,tt),V.test(a[0].type)&&t.parentNode||t))){if(a.splice(o,1),e=r.length&&dt(a),!e)return H.apply(n,q.call(r,0)),n;break}}}return s(e,p)(r,t,d,n,V.test(e)),n}i.pseudos.nth=i.pseudos.eq;function Tt(){}i.filters=Tt.prototype=i.pseudos,i.setFilters=new Tt,c(),st.attr=b.attr,b.find=st,b.expr=st.selectors,b.expr[":"]=b.expr.pseudos,b.unique=st.uniqueSort,b.text=st.getText,b.isXMLDoc=st.isXML,b.contains=st.contains}(e);var at=/Until$/,st=/^(?:parents|prev(?:Until|All))/,ut=/^.[^:#\[\.,]*$/,lt=b.expr.match.needsContext,ct={children:!0,contents:!0,next:!0,prev:!0};b.fn.extend({find:function(e){var t,n,r,i=this.length;if("string"!=typeof e)return r=this,this.pushStack(b(e).filter(function(){for(t=0;i>t;t++)if(b.contains(r[t],this))return!0}));for(n=[],t=0;i>t;t++)b.find(e,this[t],n);return n=this.pushStack(i>1?b.unique(n):n),n.selector=(this.selector?this.selector+" ":"")+e,n},has:function(e){var t,n=b(e,this),r=n.length;return this.filter(function(){for(t=0;r>t;t++)if(b.contains(this,n[t]))return!0})},not:function(e){return this.pushStack(ft(this,e,!1))},filter:function(e){return this.pushStack(ft(this,e,!0))},is:function(e){return!!e&&("string"==typeof e?lt.test(e)?b(e,this.context).index(this[0])>=0:b.filter(e,this).length>0:this.filter(e).length>0)},closest:function(e,t){var n,r=0,i=this.length,o=[],a=lt.test(e)||"string"!=typeof e?b(e,t||this.context):0;for(;i>r;r++){n=this[r];while(n&&n.ownerDocument&&n!==t&&11!==n.nodeType){if(a?a.index(n)>-1:b.find.matchesSelector(n,e)){o.push(n);break}n=n.parentNode}}return this.pushStack(o.length>1?b.unique(o):o)},index:function(e){return e?"string"==typeof e?b.inArray(this[0],b(e)):b.inArray(e.jquery?e[0]:e,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){var n="string"==typeof e?b(e,t):b.makeArray(e&&e.nodeType?[e]:e),r=b.merge(this.get(),n);return this.pushStack(b.unique(r))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),b.fn.andSelf=b.fn.addBack;function pt(e,t){do e=e[t];while(e&&1!==e.nodeType);return e}b.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return b.dir(e,"parentNode")},parentsUntil:function(e,t,n){return b.dir(e,"parentNode",n)},next:function(e){return pt(e,"nextSibling")},prev:function(e){return pt(e,"previousSibling")},nextAll:function(e){return b.dir(e,"nextSibling")},prevAll:function(e){return b.dir(e,"previousSibling")},nextUntil:function(e,t,n){return b.dir(e,"nextSibling",n)},prevUntil:function(e,t,n){return b.dir(e,"previousSibling",n)},siblings:function(e){return b.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return b.sibling(e.firstChild)},contents:function(e){return b.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:b.merge([],e.childNodes)}},function(e,t){b.fn[e]=function(n,r){var i=b.map(this,t,n);return at.test(e)||(r=n),r&&"string"==typeof r&&(i=b.filter(r,i)),i=this.length>1&&!ct[e]?b.unique(i):i,this.length>1&&st.test(e)&&(i=i.reverse()),this.pushStack(i)}}),b.extend({filter:function(e,t,n){return n&&(e=":not("+e+")"),1===t.length?b.find.matchesSelector(t[0],e)?[t[0]]:[]:b.find.matches(e,t)},dir:function(e,n,r){var i=[],o=e[n];while(o&&9!==o.nodeType&&(r===t||1!==o.nodeType||!b(o).is(r)))1===o.nodeType&&i.push(o),o=o[n];return i},sibling:function(e,t){var n=[];for(;e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n}});function ft(e,t,n){if(t=t||0,b.isFunction(t))return b.grep(e,function(e,r){var i=!!t.call(e,r,e);return i===n});if(t.nodeType)return b.grep(e,function(e){return e===t===n});if("string"==typeof t){var r=b.grep(e,function(e){return 1===e.nodeType});if(ut.test(t))return b.filter(t,r,!n);t=b.filter(t,r)}return b.grep(e,function(e){return b.inArray(e,t)>=0===n})}function dt(e){var t=ht.split("|"),n=e.createDocumentFragment();if(n.createElement)while(t.length)n.createElement(t.pop());return n}var ht="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",gt=/ jQuery\d+="(?:null|\d+)"/g,mt=RegExp("<(?:"+ht+")[\\s/>]","i"),yt=/^\s+/,vt=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bt=/<([\w:]+)/,xt=/\s*$/g,At={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:b.support.htmlSerialize?[0,"",""]:[1,"X
","
"]},jt=dt(o),Dt=jt.appendChild(o.createElement("div"));At.optgroup=At.option,At.tbody=At.tfoot=At.colgroup=At.caption=At.thead,At.th=At.td,b.fn.extend({text:function(e){return b.access(this,function(e){return e===t?b.text(this):this.empty().append((this[0]&&this[0].ownerDocument||o).createTextNode(e))},null,e,arguments.length)},wrapAll:function(e){if(b.isFunction(e))return this.each(function(t){b(this).wrapAll(e.call(this,t))});if(this[0]){var t=b(e,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstChild&&1===e.firstChild.nodeType)e=e.firstChild;return e}).append(this)}return this},wrapInner:function(e){return b.isFunction(e)?this.each(function(t){b(this).wrapInner(e.call(this,t))}):this.each(function(){var t=b(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=b.isFunction(e);return this.each(function(n){b(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(){return this.parent().each(function(){b.nodeName(this,"body")||b(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,!0,function(e){(1===this.nodeType||11===this.nodeType||9===this.nodeType)&&this.appendChild(e)})},prepend:function(){return this.domManip(arguments,!0,function(e){(1===this.nodeType||11===this.nodeType||9===this.nodeType)&&this.insertBefore(e,this.firstChild)})},before:function(){return this.domManip(arguments,!1,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return this.domManip(arguments,!1,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},remove:function(e,t){var n,r=0;for(;null!=(n=this[r]);r++)(!e||b.filter(e,[n]).length>0)&&(t||1!==n.nodeType||b.cleanData(Ot(n)),n.parentNode&&(t&&b.contains(n.ownerDocument,n)&&Mt(Ot(n,"script")),n.parentNode.removeChild(n)));return this},empty:function(){var e,t=0;for(;null!=(e=this[t]);t++){1===e.nodeType&&b.cleanData(Ot(e,!1));while(e.firstChild)e.removeChild(e.firstChild);e.options&&b.nodeName(e,"select")&&(e.options.length=0)}return this},clone:function(e,t){return e=null==e?!1:e,t=null==t?e:t,this.map(function(){return b.clone(this,e,t)})},html:function(e){return b.access(this,function(e){var n=this[0]||{},r=0,i=this.length;if(e===t)return 1===n.nodeType?n.innerHTML.replace(gt,""):t;if(!("string"!=typeof e||Tt.test(e)||!b.support.htmlSerialize&&mt.test(e)||!b.support.leadingWhitespace&&yt.test(e)||At[(bt.exec(e)||["",""])[1].toLowerCase()])){e=e.replace(vt,"<$1>");try{for(;i>r;r++)n=this[r]||{},1===n.nodeType&&(b.cleanData(Ot(n,!1)),n.innerHTML=e);n=0}catch(o){}}n&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(e){var t=b.isFunction(e);return t||"string"==typeof e||(e=b(e).not(this).detach()),this.domManip([e],!0,function(e){var t=this.nextSibling,n=this.parentNode;n&&(b(this).remove(),n.insertBefore(e,t))})},detach:function(e){return this.remove(e,!0)},domManip:function(e,n,r){e=f.apply([],e);var i,o,a,s,u,l,c=0,p=this.length,d=this,h=p-1,g=e[0],m=b.isFunction(g);if(m||!(1>=p||"string"!=typeof g||b.support.checkClone)&&Ct.test(g))return this.each(function(i){var o=d.eq(i);m&&(e[0]=g.call(this,i,n?o.html():t)),o.domManip(e,n,r)});if(p&&(l=b.buildFragment(e,this[0].ownerDocument,!1,this),i=l.firstChild,1===l.childNodes.length&&(l=i),i)){for(n=n&&b.nodeName(i,"tr"),s=b.map(Ot(l,"script"),Ht),a=s.length;p>c;c++)o=l,c!==h&&(o=b.clone(o,!0,!0),a&&b.merge(s,Ot(o,"script"))),r.call(n&&b.nodeName(this[c],"table")?Lt(this[c],"tbody"):this[c],o,c);if(a)for(u=s[s.length-1].ownerDocument,b.map(s,qt),c=0;a>c;c++)o=s[c],kt.test(o.type||"")&&!b._data(o,"globalEval")&&b.contains(u,o)&&(o.src?b.ajax({url:o.src,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0}):b.globalEval((o.text||o.textContent||o.innerHTML||"").replace(St,"")));l=i=null}return this}});function Lt(e,t){return e.getElementsByTagName(t)[0]||e.appendChild(e.ownerDocument.createElement(t))}function Ht(e){var t=e.getAttributeNode("type");return e.type=(t&&t.specified)+"/"+e.type,e}function qt(e){var t=Et.exec(e.type);return t?e.type=t[1]:e.removeAttribute("type"),e}function Mt(e,t){var n,r=0;for(;null!=(n=e[r]);r++)b._data(n,"globalEval",!t||b._data(t[r],"globalEval"))}function _t(e,t){if(1===t.nodeType&&b.hasData(e)){var n,r,i,o=b._data(e),a=b._data(t,o),s=o.events;if(s){delete a.handle,a.events={};for(n in s)for(r=0,i=s[n].length;i>r;r++)b.event.add(t,n,s[n][r])}a.data&&(a.data=b.extend({},a.data))}}function Ft(e,t){var n,r,i;if(1===t.nodeType){if(n=t.nodeName.toLowerCase(),!b.support.noCloneEvent&&t[b.expando]){i=b._data(t);for(r in i.events)b.removeEvent(t,r,i.handle);t.removeAttribute(b.expando)}"script"===n&&t.text!==e.text?(Ht(t).text=e.text,qt(t)):"object"===n?(t.parentNode&&(t.outerHTML=e.outerHTML),b.support.html5Clone&&e.innerHTML&&!b.trim(t.innerHTML)&&(t.innerHTML=e.innerHTML)):"input"===n&&Nt.test(e.type)?(t.defaultChecked=t.checked=e.checked,t.value!==e.value&&(t.value=e.value)):"option"===n?t.defaultSelected=t.selected=e.defaultSelected:("input"===n||"textarea"===n)&&(t.defaultValue=e.defaultValue)}}b.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,t){b.fn[e]=function(e){var n,r=0,i=[],o=b(e),a=o.length-1;for(;a>=r;r++)n=r===a?this:this.clone(!0),b(o[r])[t](n),d.apply(i,n.get());return this.pushStack(i)}});function Ot(e,n){var r,o,a=0,s=typeof e.getElementsByTagName!==i?e.getElementsByTagName(n||"*"):typeof e.querySelectorAll!==i?e.querySelectorAll(n||"*"):t;if(!s)for(s=[],r=e.childNodes||e;null!=(o=r[a]);a++)!n||b.nodeName(o,n)?s.push(o):b.merge(s,Ot(o,n));return n===t||n&&b.nodeName(e,n)?b.merge([e],s):s}function Bt(e){Nt.test(e.type)&&(e.defaultChecked=e.checked)}b.extend({clone:function(e,t,n){var r,i,o,a,s,u=b.contains(e.ownerDocument,e);if(b.support.html5Clone||b.isXMLDoc(e)||!mt.test("<"+e.nodeName+">")?o=e.cloneNode(!0):(Dt.innerHTML=e.outerHTML,Dt.removeChild(o=Dt.firstChild)),!(b.support.noCloneEvent&&b.support.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||b.isXMLDoc(e)))for(r=Ot(o),s=Ot(e),a=0;null!=(i=s[a]);++a)r[a]&&Ft(i,r[a]);if(t)if(n)for(s=s||Ot(e),r=r||Ot(o),a=0;null!=(i=s[a]);a++)_t(i,r[a]);else _t(e,o);return r=Ot(o,"script"),r.length>0&&Mt(r,!u&&Ot(e,"script")),r=s=i=null,o},buildFragment:function(e,t,n,r){var i,o,a,s,u,l,c,p=e.length,f=dt(t),d=[],h=0;for(;p>h;h++)if(o=e[h],o||0===o)if("object"===b.type(o))b.merge(d,o.nodeType?[o]:o);else if(wt.test(o)){s=s||f.appendChild(t.createElement("div")),u=(bt.exec(o)||["",""])[1].toLowerCase(),c=At[u]||At._default,s.innerHTML=c[1]+o.replace(vt,"<$1>")+c[2],i=c[0];while(i--)s=s.lastChild;if(!b.support.leadingWhitespace&&yt.test(o)&&d.push(t.createTextNode(yt.exec(o)[0])),!b.support.tbody){o="table"!==u||xt.test(o)?""!==c[1]||xt.test(o)?0:s:s.firstChild,i=o&&o.childNodes.length;while(i--)b.nodeName(l=o.childNodes[i],"tbody")&&!l.childNodes.length&&o.removeChild(l) +}b.merge(d,s.childNodes),s.textContent="";while(s.firstChild)s.removeChild(s.firstChild);s=f.lastChild}else d.push(t.createTextNode(o));s&&f.removeChild(s),b.support.appendChecked||b.grep(Ot(d,"input"),Bt),h=0;while(o=d[h++])if((!r||-1===b.inArray(o,r))&&(a=b.contains(o.ownerDocument,o),s=Ot(f.appendChild(o),"script"),a&&Mt(s),n)){i=0;while(o=s[i++])kt.test(o.type||"")&&n.push(o)}return s=null,f},cleanData:function(e,t){var n,r,o,a,s=0,u=b.expando,l=b.cache,p=b.support.deleteExpando,f=b.event.special;for(;null!=(n=e[s]);s++)if((t||b.acceptData(n))&&(o=n[u],a=o&&l[o])){if(a.events)for(r in a.events)f[r]?b.event.remove(n,r):b.removeEvent(n,r,a.handle);l[o]&&(delete l[o],p?delete n[u]:typeof n.removeAttribute!==i?n.removeAttribute(u):n[u]=null,c.push(o))}}});var Pt,Rt,Wt,$t=/alpha\([^)]*\)/i,It=/opacity\s*=\s*([^)]*)/,zt=/^(top|right|bottom|left)$/,Xt=/^(none|table(?!-c[ea]).+)/,Ut=/^margin/,Vt=RegExp("^("+x+")(.*)$","i"),Yt=RegExp("^("+x+")(?!px)[a-z%]+$","i"),Jt=RegExp("^([+-])=("+x+")","i"),Gt={BODY:"block"},Qt={position:"absolute",visibility:"hidden",display:"block"},Kt={letterSpacing:0,fontWeight:400},Zt=["Top","Right","Bottom","Left"],en=["Webkit","O","Moz","ms"];function tn(e,t){if(t in e)return t;var n=t.charAt(0).toUpperCase()+t.slice(1),r=t,i=en.length;while(i--)if(t=en[i]+n,t in e)return t;return r}function nn(e,t){return e=t||e,"none"===b.css(e,"display")||!b.contains(e.ownerDocument,e)}function rn(e,t){var n,r,i,o=[],a=0,s=e.length;for(;s>a;a++)r=e[a],r.style&&(o[a]=b._data(r,"olddisplay"),n=r.style.display,t?(o[a]||"none"!==n||(r.style.display=""),""===r.style.display&&nn(r)&&(o[a]=b._data(r,"olddisplay",un(r.nodeName)))):o[a]||(i=nn(r),(n&&"none"!==n||!i)&&b._data(r,"olddisplay",i?n:b.css(r,"display"))));for(a=0;s>a;a++)r=e[a],r.style&&(t&&"none"!==r.style.display&&""!==r.style.display||(r.style.display=t?o[a]||"":"none"));return e}b.fn.extend({css:function(e,n){return b.access(this,function(e,n,r){var i,o,a={},s=0;if(b.isArray(n)){for(o=Rt(e),i=n.length;i>s;s++)a[n[s]]=b.css(e,n[s],!1,o);return a}return r!==t?b.style(e,n,r):b.css(e,n)},e,n,arguments.length>1)},show:function(){return rn(this,!0)},hide:function(){return rn(this)},toggle:function(e){var t="boolean"==typeof e;return this.each(function(){(t?e:nn(this))?b(this).show():b(this).hide()})}}),b.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Wt(e,"opacity");return""===n?"1":n}}}},cssNumber:{columnCount:!0,fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":b.support.cssFloat?"cssFloat":"styleFloat"},style:function(e,n,r,i){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var o,a,s,u=b.camelCase(n),l=e.style;if(n=b.cssProps[u]||(b.cssProps[u]=tn(l,u)),s=b.cssHooks[n]||b.cssHooks[u],r===t)return s&&"get"in s&&(o=s.get(e,!1,i))!==t?o:l[n];if(a=typeof r,"string"===a&&(o=Jt.exec(r))&&(r=(o[1]+1)*o[2]+parseFloat(b.css(e,n)),a="number"),!(null==r||"number"===a&&isNaN(r)||("number"!==a||b.cssNumber[u]||(r+="px"),b.support.clearCloneStyle||""!==r||0!==n.indexOf("background")||(l[n]="inherit"),s&&"set"in s&&(r=s.set(e,r,i))===t)))try{l[n]=r}catch(c){}}},css:function(e,n,r,i){var o,a,s,u=b.camelCase(n);return n=b.cssProps[u]||(b.cssProps[u]=tn(e.style,u)),s=b.cssHooks[n]||b.cssHooks[u],s&&"get"in s&&(a=s.get(e,!0,r)),a===t&&(a=Wt(e,n,i)),"normal"===a&&n in Kt&&(a=Kt[n]),""===r||r?(o=parseFloat(a),r===!0||b.isNumeric(o)?o||0:a):a},swap:function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];i=n.apply(e,r||[]);for(o in t)e.style[o]=a[o];return i}}),e.getComputedStyle?(Rt=function(t){return e.getComputedStyle(t,null)},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),u=s?s.getPropertyValue(n)||s[n]:t,l=e.style;return s&&(""!==u||b.contains(e.ownerDocument,e)||(u=b.style(e,n)),Yt.test(u)&&Ut.test(n)&&(i=l.width,o=l.minWidth,a=l.maxWidth,l.minWidth=l.maxWidth=l.width=u,u=s.width,l.width=i,l.minWidth=o,l.maxWidth=a)),u}):o.documentElement.currentStyle&&(Rt=function(e){return e.currentStyle},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),u=s?s[n]:t,l=e.style;return null==u&&l&&l[n]&&(u=l[n]),Yt.test(u)&&!zt.test(n)&&(i=l.left,o=e.runtimeStyle,a=o&&o.left,a&&(o.left=e.currentStyle.left),l.left="fontSize"===n?"1em":u,u=l.pixelLeft+"px",l.left=i,a&&(o.left=a)),""===u?"auto":u});function on(e,t,n){var r=Vt.exec(t);return r?Math.max(0,r[1]-(n||0))+(r[2]||"px"):t}function an(e,t,n,r,i){var o=n===(r?"border":"content")?4:"width"===t?1:0,a=0;for(;4>o;o+=2)"margin"===n&&(a+=b.css(e,n+Zt[o],!0,i)),r?("content"===n&&(a-=b.css(e,"padding"+Zt[o],!0,i)),"margin"!==n&&(a-=b.css(e,"border"+Zt[o]+"Width",!0,i))):(a+=b.css(e,"padding"+Zt[o],!0,i),"padding"!==n&&(a+=b.css(e,"border"+Zt[o]+"Width",!0,i)));return a}function sn(e,t,n){var r=!0,i="width"===t?e.offsetWidth:e.offsetHeight,o=Rt(e),a=b.support.boxSizing&&"border-box"===b.css(e,"boxSizing",!1,o);if(0>=i||null==i){if(i=Wt(e,t,o),(0>i||null==i)&&(i=e.style[t]),Yt.test(i))return i;r=a&&(b.support.boxSizingReliable||i===e.style[t]),i=parseFloat(i)||0}return i+an(e,t,n||(a?"border":"content"),r,o)+"px"}function un(e){var t=o,n=Gt[e];return n||(n=ln(e,t),"none"!==n&&n||(Pt=(Pt||b("