feat: detiler compitability (#26)

* fix: multiple bugs and unexpected behaviors caused by syncing upstream

* fix: multiple bugs

* fix: env variables compatible with osm2pgsql v1.6.0

* feat: added application metrics

* fix: added metrics helm values

* fix: minor tweaks

* feat: detiler compitability

* fix: dockerfile osm2pgsql commit sha

* fix(configurations): dockerfile library dependency

* fix: integration bugs

Author: melancholiai

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:20.04 as build
 
 ENV DEBIAN_FRONTEND=noninteractive
 ARG OSM2PGSQL_REPOSITORY=https://github.com/MapColonies/osm2pgsql.git
-ARG OSM2PGSQL_COMMIT_SHA=6c68ead630575bc2545cd668c6e6dc7b6d4772f9
+ARG OSM2PGSQL_COMMIT_SHA=078884e01c1dc3c9c20c85cf9e57436b294e7e65
 ARG OSMIUM_TOOL_TAG=v1.16.0
 ARG PROTOZERO_TAG=v1.7.1
 ARG LIBOSMIUM_TAG=v2.20.0
@@ -74,7 +74,7 @@ RUN ln -s /osm2pgsql/osm2pgsql /bin/osm2pgsql && ln -s /osmium-tool/build/osmium
 RUN apt-get update \
     && apt-get -yq install curl \
     && curl -L https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash \
-    && apt-get -yq install nodejs libboost-filesystem-dev libpq-dev libproj-dev liblua5.3-dev libboost-program-options-dev
+    && apt-get -yq install nodejs libboost-filesystem-dev libpq-dev libproj-dev liblua5.3-dev libboost-program-options-dev libexpat1-dev
 
 COPY ./package*.json ./
 

src/cli/commands/append/appendFactory.ts

@@ -45,11 +45,14 @@ export const appendCommandFactory: FactoryFunction<CommandModule<GlobalArguments
       })
       .option('uploadTargets', {
         alias: ['u', 'upload-targets'],
-        type: 'string',
+        type: 'array',
         describe: 'upload expired tiles to the selected sources',
         choices: ['s3', 'queue'],
-        array: true,
+        string: true,
         default: [] as string[],
+        coerce: (targetsString: string) => {
+          return targetsString[0] ? targetsString[0].split(',') : [];
+        },
       })
       .option('name', {
         alias: ['queue-name'],
@@ -89,7 +92,7 @@ export const appendCommandFactory: FactoryFunction<CommandModule<GlobalArguments
       const configContent = await fsPromises.readFile(config, 'utf-8');
       const appendEntities = JSON.parse(configContent) as AppendEntity[];
 
-      logger.debug({ msg: 'append configuration', projectId: s3ProjectId, entitiesCount: appendEntities.length });
+      logger.info({ msg: 'append configuration', projectId: s3ProjectId, entitiesCount: appendEntities.length, uploadTargets });
 
       await manager.prepareManager(s3ProjectId, appendEntities, uploadTargets as ExpireTilesUploadTarget[]);
 

src/cli/commands/append/appendManager.ts

@@ -146,6 +146,8 @@ export class AppendManager {
     try {
       await this.appendNextState(replicationUrl);
 
+      await this.stateTracker.updateRemoteTimestamp();
+
       if (this.shouldGenerateExpireOutput) {
         await this.uploadExpired();
       }
@@ -204,6 +206,8 @@ export class AppendManager {
       try {
         await this.appendNextState(replicationUrl);
 
+        await this.stateTracker.updateRemoteTimestamp();
+
         if (this.shouldGenerateExpireOutput) {
           await this.uploadExpired();
         }
@@ -351,6 +355,7 @@ export class AppendManager {
         entityId: entity.id,
         localExpireTilesListPath,
         expireListCount: expireList.length,
+        state: this.stateTracker.nextState,
       });
 
       for await (const target of this.uploadTargets) {
@@ -412,6 +417,7 @@ export class AppendManager {
     const payload: TileRequestQueuePayload = {
       items: expiredTilesBbox.map((bbox) => ({ area: bbox, minZoom, maxZoom })),
       source: 'expiredTiles',
+      state: this.stateTracker.nextState,
     };
 
     await this.pushPayloadToQueue(payload);

src/cli/commands/append/interfaces.ts

@@ -31,4 +31,6 @@ export interface TileRequestQueuePayloadItem<A = BoundingBox | Feature> extends
 export interface TileRequestQueuePayload<A = BoundingBox | Feature> {
   items: TileRequestQueuePayloadItem<A>[];
   source: 'expiredTiles';
+  state?: number;
+  force?: boolean;
 }

src/cli/commands/append/stateTracker.ts

@@ -1,7 +1,7 @@
 import { join } from 'path';
 import { inject, singleton } from 'tsyringe';
 import { Logger } from '@map-colonies/js-logger';
-import { DATA_DIR, DEFAULT_SEQUENCE_NUMBER, SEQUENCE_NUMBER_REGEX, SERVICES, STATE_FILE } from '../../../common/constants';
+import { DATA_DIR, DEFAULT_SEQUENCE_NUMBER, SEQUENCE_NUMBER_REGEX, SERVICES, STATE_FILE, TIMESTAMP_REGEX } from '../../../common/constants';
 import { BucketDoesNotExistError, InvalidStateFileError } from '../../../common/errors';
 import { createDirectory, fetchSequenceNumber, streamToString } from '../../../common/util';
 import { ReplicationClient } from '../../../httpClient/replicationClient';
@@ -95,6 +95,29 @@ export class StateTracker {
     this.totalAppends++;
   }
 
+  public async updateRemoteTimestamp(): Promise<void> {
+    const updatedTimestamp = new Date()
+      .toISOString()
+      .replace(/\.\d+Z$/, 'Z')
+      .replace(/:/g, '\\:');
+
+    this.logger.info({
+      msg: 'attempting to update remote timestamp on configured bucket',
+      projectId: this.projectId,
+      currentState: this.current,
+      toState: this.nextState,
+      bucketName: this.s3Client.bucketName,
+      acl: this.s3Client.acl,
+      updatedTimestamp,
+    });
+
+    stateContent = stateContent.replace(TIMESTAMP_REGEX, `timestamp=${updatedTimestamp}`);
+    const stateBuffer = Buffer.from(stateContent, 'utf-8');
+    const stateKey = join(this.projectId, STATE_FILE);
+
+    await this.s3Client.putObjectWrapper(stateKey, stateBuffer);
+  }
+
   private fetchSequenceNumberSafely(content: string): number {
     try {
       return fetchSequenceNumber(content);

src/cli/middlewares/index.ts

@@ -18,10 +18,10 @@ type RegisterOnContainerMiddlewareFactory<T> = (container: DependencyContainer)
 
 export const s3RegistrationMiddlewareFactory: RegisterOnContainerMiddlewareFactory<GlobalArguments> = (dependencyContainer) => {
   const middleware = (args: Arguments<GlobalArguments>): void => {
-    const { s3Endpoint, s3BucketName } = args;
+    const { s3Endpoint, s3BucketName, s3Acl } = args;
 
     const configStore = dependencyContainer.resolve<ConfigStore>(SERVICES.CONFIG_STORE);
-    configStore.set('s3', { bucketName: s3BucketName });
+    configStore.set('s3', { bucketName: s3BucketName, acl: s3Acl });
 
     const client = new S3Client({
       endpoint: s3Endpoint,
@@ -89,11 +89,6 @@ export const uploadTargetsRegistrationMiddlewareFactory: RegisterOnContainerMidd
           },
         ]);
       }
-
-      if (uploadTargetsSet.has('s3')) {
-        const { s3Acl } = args;
-        configStore.set('s3.acl', s3Acl);
-      }
     }
   };
 

src/common/constants.ts

@@ -53,6 +53,7 @@ export const DEFAULT_SEQUENCE_NUMBER = -1;
 export const DEFAULT_PROJECT_CREATION_STATE = 1;
 export const SEQUENCE_NUMBER = 'sequenceNumber';
 export const SEQUENCE_NUMBER_REGEX = /sequenceNumber=\d+/;
+export const TIMESTAMP_REGEX = /timestamp=\d{4}-\d{2}-\d{2}T\d{2}\\?:\d{2}\\?:\d{2}Z/;
 export const SEQUENCE_NUMBER_PADDING_AMOUNT = 3;
 export const DIFF_TOP_DIR_DIVIDER = 1000000;
 export const DIFF_BOTTOM_DIR_DIVIDER = 1000;