iso week fix (#373)

ivke995 · Ante-Koceic · web-flow · commit abf13bdf5200 · 2024-01-15T16:29:17.000+01:00
* started

* Using Date.UTC to get currentDate when seeding

* Changed week to text

* date map changed

* utc fixed

* test test teeest

* unused import removed

* Fixed errors from conflicts

* new db sql file generated

---------

Co-authored-by: Ante Koceić &lt;ante@crocoder.dev&gt;
diff --git a/migrations/tenant-db/0000_gray_moondragon.sql b/migrations/tenant-db/0000_gray_moondragon.sql
@@ -160,7 +160,7 @@ CREATE TABLE `extract_timeline_events` (
 CREATE TABLE `transform_dates` (
 	`id` integer PRIMARY KEY NOT NULL,
 	`day` integer NOT NULL,
-	`week` integer NOT NULL,
+	`week` text NOT NULL,
 	`month` integer NOT NULL,
 	`year` integer NOT NULL,
 	`__created_at` integer DEFAULT (strftime('%s', 'now')),
diff --git a/migrations/tenant-db/meta/0000_snapshot.json b/migrations/tenant-db/meta/0000_snapshot.json
@@ -1,7 +1,7 @@
 {
   "version": "5",
   "dialect": "sqlite",
-  "id": "5aea1665-0130-4769-b328-0cf598573a17",
+  "id": "5486d1de-b91c-4ff9-80fa-cbb0481ad6e8",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "crawl_events": {
@@ -1034,7 +1034,8 @@
           "columns": [
             "member_id",
             "repository_id"
-          ]
+          ],
+          "name": "extract_repositories_to_members_repository_id_member_id_pk"
         }
       },
       "uniqueConstraints": {}
@@ -1170,7 +1171,7 @@
         },
         "week": {
           "name": "week",
-          "type": "integer",
+          "type": "text",
           "primaryKey": false,
           "notNull": true,
           "autoincrement": false
diff --git a/migrations/tenant-db/meta/_journal.json b/migrations/tenant-db/meta/_journal.json
@@ -5,8 +5,8 @@
     {
       "idx": 0,
       "version": "5",
-      "when": 1704818265547,
-      "tag": "0000_bitter_firelord",
+      "when": 1705312108477,
+      "tag": "0000_gray_moondragon",
       "breakpoints": true
     }
   ]
diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts
@@ -7,6 +7,7 @@ import { parseHunks } from './parse-hunks';
 import { type SQLiteTransaction } from 'drizzle-orm/sqlite-core';
 import { type ResultSet } from '@libsql/client/.';
 import { isMemberKnownBot } from './known-bots';
+import { getDateInfo } from '../../../schemas/transform/src/seed/dimensions';
 
 type BrandedDatabase<T> = LibSQLDatabase<Record<string, never>> & { __brand: T }
 
@@ -168,7 +169,7 @@ type DMY = {
   year: number,
   month: number,
   day: number,
-  week: number,
+  week: string,
 };
 
 type selectDatesArgs = {
@@ -181,21 +182,11 @@ type selectDatesArgs = {
   lastUpdatedAt: DMY | null,
 };
 
-function getWeek(date: Date): number {
-  // Logic copied from dimensions.ts
-  return Math.ceil(((+date - +new Date(date.getUTCFullYear(), 0, 1)) / (24 * 60 * 60 * 1000)) / 7);
-}
-
 function getDMY(date: Date | null) {
   if (date === null) {
     return null;
   }
-  return {
-    year: date.getUTCFullYear(),
-    month: date.getUTCMonth() + 1,
-    day: date.getUTCDate(),
-    week: getWeek(date),
-  };
+  return getDateInfo(new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())))
 }
 
 async function mapDatesToTransformedDates(db: TransformDatabase, dates: mapDatesToTransformedDatesArgs, nullDateId: number) {
@@ -217,7 +208,6 @@ function getDMYQuery(dmy: DMY | null) {
   if (dmy === null) {
     return undefined;
   }
-
   return and(
     eq(transform.dates.year, dmy.year),
     eq(transform.dates.month, dmy.month),
@@ -231,7 +221,7 @@ function getDateIdOrNullDateId(dmy: DMY | null, datesData: {
   year: number;
   month: number;
   day: number;
-  week: number;
+  week: string;
 }[], nullDateId: number) {
   if (dmy === null) {
     return {
diff --git a/packages/schemas/transform/src/dates.ts b/packages/schemas/transform/src/dates.ts
@@ -1,12 +1,12 @@
 import type { InferSelectModel, InferInsertModel } from 'drizzle-orm';
 import { sql } from 'drizzle-orm';
-import { integer, uniqueIndex } from 'drizzle-orm/sqlite-core';
+import { text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core';
 import { sqliteTable } from './transform-table';
 
 export const dates = sqliteTable('dates', {
   id: integer('id').primaryKey(),
   day: integer('day').notNull(),
-  week: integer('week').notNull(),
+  week: text('week').notNull(),
   month: integer('month').notNull(),
   year: integer('year').notNull(),
 
diff --git a/packages/schemas/transform/src/seed/dimensions.test.ts b/packages/schemas/transform/src/seed/dimensions.test.ts
@@ -4,11 +4,12 @@ import { createClient } from "@libsql/client";
 import { drizzle, type LibSQLDatabase } from "drizzle-orm/libsql";
 import { migrate } from "drizzle-orm/libsql/migrator";
 
-import { seed } from "./dimensions";
+import { getDateInfo, seed } from "./dimensions";
 import { forgeUsers } from "../forge-users";
 import { dates } from "../dates";
 import { mergeRequests } from "../merge-requests";
 import { repositories } from "../repositories";
+import { getFirstDay } from "./dimensions"
 
 let db: LibSQLDatabase;
 let sqlite: ReturnType<typeof createClient>;
@@ -66,4 +67,48 @@ describe("dimensions", () => {
       }))
     });
   });
+  describe("getFirstDay", () => {
+    test("should return first day of iso year", () => {
+      const year1980 = getFirstDay(1980);
+      const year1981 = getFirstDay(1981);
+      const year2015 = getFirstDay(2015);
+      const year2022 = getFirstDay(2022);
+      expect(year1980).toEqual(new Date("1979-12-31"));
+      expect(year1981).toEqual(new Date("1980-12-29"));
+      expect(year2015).toEqual(new Date("2014-12-29"));
+      expect(year2022).toEqual(new Date("2022-01-03"));
+    })
+  });
+  describe("getDateInfo", () => {
+    test("should return correct data for given date", () => {
+      const date1 = getDateInfo(new Date(Date.UTC(2021, 0, 4)));
+      const date2 = getDateInfo(new Date(Date.UTC(2021, 0, 3)));
+      const date3 = getDateInfo(new Date(Date.UTC(2023, 0, 4)));
+      const date4 = getDateInfo(new Date(Date.UTC(2023, 0, 3)));
+      expect(date1).toEqual({
+        day: 4,
+        week: "2021-W1",
+        month: 1,
+        year: 2021,
+      });
+      expect(date2).toEqual({
+        day: 3,
+        week: "2020-W53",
+        month: 1,
+        year: 2021,
+      });
+      expect(date3).toEqual({
+        day: 4,
+        week: "2023-W1",
+        month: 1,
+        year: 2023,
+      });
+      expect(date4).toEqual({
+        day: 3,
+        week: "2023-W1",
+        month: 1,
+        year: 2023,
+      });
+    });
+  });
 });
diff --git a/packages/schemas/transform/src/seed/dimensions.ts b/packages/schemas/transform/src/seed/dimensions.ts
@@ -20,7 +20,7 @@ const nullForgeUser = {
 const nullDate = {
   id: 1,
   day: Number.MAX_SAFE_INTEGER,
-  week: Number.MAX_SAFE_INTEGER,
+  week: '',
   month: Number.MAX_SAFE_INTEGER,
   year: Number.MAX_SAFE_INTEGER,
 } satisfies NewTransformDate;
@@ -61,17 +61,58 @@ export async function seed(db: LibSQLDatabase, startDate: Date, endDate: Date):
 
 }
 
+export function getFirstDay(year: number): Date {
+  let firstDayOfYear = new Date(Date.UTC(year, 0, 1));
+  if (firstDayOfYear.getUTCDay() !== 1) {
+    for (let i = 1; i < 4; i++) {
+      const p = new Date(firstDayOfYear);
+      const n = new Date(firstDayOfYear);
+      p.setUTCDate(p.getUTCDate() - i);
+      n.setUTCDate(n.getUTCDate() + i);
+      if (p.getUTCDay() === 1) {
+        firstDayOfYear = p;
+        break;
+      } else if (n.getUTCDay() === 1) {
+        firstDayOfYear = n;
+        break;
+      }
+    }
+  }
+  return firstDayOfYear;
+}
+
+export function checkWeek(week: number, year: number): { newWeek: string } {
+  let isoWeek = week;
+  let isoYear = year;
+  if (week < 1) {
+    const lastDayOfPrev = new Date(Date.UTC(year - 1, 11, 31));
+    const firstDayOfPrev = getFirstDay(year - 1);
+    isoWeek = Math.ceil(((lastDayOfPrev.getTime() - firstDayOfPrev.getTime()) / (24 * 60 * 60 * 1000) + 1) / 7)
+    isoYear = isoYear - 1
+  }
+  return { newWeek: `${isoYear}-W${isoWeek}` };
+}
+
+export function getDateInfo(date: Date): {day: number, week: string, month: number, year: number} {
+ 
+  const firstDay = getFirstDay(date.getUTCFullYear());
+  const week = Math.ceil(((date.getTime() - firstDay.getTime()) / (24 * 60 * 60 * 1000) + 1) / 7);
+  const { newWeek } = checkWeek(week, date.getUTCFullYear());
+  return {
+    day: date.getUTCDate(),
+    week: newWeek,
+    month: date.getUTCMonth() + 1, // Months are zero-based, so we add 1.
+    year: date.getUTCFullYear(),
+ }
+}
+
 function generateDates(startDate: Date, endDate: Date) {
   const dates = [];
-  const currentDate = new Date(startDate);
+  const currentDate = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getDate()));
 
   while (currentDate <= endDate) {
-    const customDate = {
-      day: currentDate.getUTCDate(),
-      week: Math.ceil(((+currentDate - +new Date(currentDate.getUTCFullYear(), 0, 1)) / (24 * 60 * 60 * 1000)) / 7),
-      month: currentDate.getUTCMonth() + 1, // Months are zero-based, so we add 1.
-      year: currentDate.getUTCFullYear(),
-    };
+
+    const customDate = getDateInfo(currentDate);
 
     dates.push(customDate);
     currentDate.setUTCDate(currentDate.getUTCDate() + 1);
diff --git a/scripts/seed.ts b/scripts/seed.ts
@@ -19,6 +19,6 @@ seed(
   drizzle(
     createClient({ url: process.env.TENANT_DATABASE_URL, authToken: process.env.TENANT_DATABASE_AUTH_TOKEN })
   ),
-  new Date(now.getTime() - 6 * MONTH),
-  new Date(now.getTime() + 6 * MONTH)
+  new Date(now.getTime() - 18 * MONTH),
+  new Date(now.getTime() + 2 * MONTH)
 ).catch(console.error);

Original file line number	Diff line number	Diff line change
`@@ -5,8 +5,8 @@`
`5`	`5`	`{`
`6`	`6`	`"idx": 0,`
`7`	`7`	`"version": "5",`
`8`		`- "when": 1704818265547,`
`9`		`- "tag": "0000_bitter_firelord",`
	`8`	`+ "when": 1705312108477,`
	`9`	`+ "tag": "0000_gray_moondragon",`
`10`	`10`	`"breakpoints": true`
`11`	`11`	`}`
`12`	`12`	`]`