Vector DB

.yarn/versions/4bce8ecc.yml

@@ -20,5 +20,6 @@ undecided:
   - "@snickerdoodlelabs/synamint-extension-sdk"
   - "@snickerdoodlelabs/core-test-harness"
   - "@snickerdoodlelabs/utils"
+  - "@snickerdoodlelabs/vector-db"
   - "@snickerdoodlelabs/web-integration"
   - "@snickerdoodlelabs/web-integration-test"

packages/core/src/implementations/SnickerdoodleCore.module.ts

@@ -29,6 +29,7 @@ import {
   ICryptoUtilsType,
 } from "@snickerdoodlelabs/node-utils";
 import {
+  IIndexedDB,
   ITokenPriceRepository,
   ITokenPriceRepositoryType,
 } from "@snickerdoodlelabs/objects";
@@ -60,6 +61,10 @@ import {
   IPersistenceContextProviderType,
   NullCloudStorage,
   INullCloudStorageType,
+  DBContextProvider,
+  IIndexedDBContextProvider,
+  IIndexedDBContextProviderType,
+  IIndexedDBContext,
 } from "@snickerdoodlelabs/persistence";
 import {
   IQueryObjectFactory,
@@ -325,8 +330,8 @@ export const snickerdoodleCoreModule = new ContainerModule(
       .to(MonitoringService)
       .inSingletonScope();
     bind<IQuestionnaireService>(IQuestionnaireServiceType)
-    .to(QuestionnaireService)
-    .inSingletonScope();
+      .to(QuestionnaireService)
+      .inSingletonScope();
     bind<IDiscordService>(IDiscordServiceType)
       .to(DiscordService)
       .inSingletonScope();
@@ -462,6 +467,16 @@ export const snickerdoodleCoreModule = new ContainerModule(
       IPersistenceContextProviderType,
     ).toConstantValue(contextProvider);
 
+    const dbContextProvider = new DBContextProvider(
+      new LogUtils(),
+      new TimeUtils(),
+      new VolatileStorageSchemaProvider(configProvider),
+    );
+
+    bind<IIndexedDBContextProvider>(
+      IIndexedDBContextProviderType,
+    ).toConstantValue(dbContextProvider);
+
     bind<IBlockchainProvider>(IBlockchainProviderType)
       .to(BlockchainProvider)
       .inSingletonScope();
@@ -543,6 +558,11 @@ export const snickerdoodleCoreModule = new ContainerModule(
       .to(DropboxCloudStorage)
       .inSingletonScope();
 
+    // bind<IIndexedDB>(IIndexedDBType).to(IndexedDB).inSingletonScope();
+    // bind<IIndexedDBContext>(IIndexedDBConfigProviderType)
+    //   .to(IIndexed)
+    //   .inSingletonScope();
+
     /**
      * Binding of Modules With Extra Capabilities.
      *

packages/core/src/implementations/SnickerdoodleCore.ts

@@ -92,9 +92,13 @@ import {
   URLString,
   INftMethods,
   IQuestionnaireMethods,
+  IVectorQuantizationMethods,
   NewQuestionnaireAnswer,
   JSONString,
   EExternalFieldKey,
+  ERecordKey,
+  IIndexedDB,
+  IIndexedDBType,
 } from "@snickerdoodlelabs/objects";
 import {
   IndexedDBVolatileStorage,
@@ -108,6 +112,11 @@ import {
   IStorageUtilsType,
   LocalStorageUtils,
 } from "@snickerdoodlelabs/utils";
+import {
+  IQuantizationService,
+  IQuantizationServiceType,
+  VectorDB,
+} from "@snickerdoodlelabs/vector-db";
 import { ethers } from "ethers";
 import { Container } from "inversify";
 import { ResultAsync } from "neverthrow";
@@ -186,6 +195,7 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
   public storage: IStorageMethods;
   public nft: INftMethods;
   public questionnaire: IQuestionnaireMethods;
+  public quantization: IVectorQuantizationMethods;
 
   public constructor(
     configOverrides?: IConfigOverrides,
@@ -220,6 +230,11 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
         .inSingletonScope();
     }
 
+    this.iocContainer
+      .bind(IQuantizationServiceType)
+      .to(VectorDB)
+      .inSingletonScope();
+
     // Setup the config
     if (configOverrides != null) {
       const configProvider =
@@ -718,6 +733,57 @@ export class SnickerdoodleCore implements ISnickerdoodleCore {
       },
     };
 
+    // Vector DB Methods --------------------------------------------------------------------
+    this.quantization = {
+      // discovers data in db
+      // way simpler, inject dependencies, iterate local db
+      initialize: (template?: IIndexedDB) => {
+        const quantizationService = this.iocContainer.get<IQuantizationService>(
+          IQuantizationServiceType,
+        );
+
+        return quantizationService.initialize(template);
+      },
+
+      table: (name: string) => {
+        const quantizationService = this.iocContainer.get<IQuantizationService>(
+          IQuantizationServiceType,
+        );
+
+        return quantizationService.table(name);
+      },
+
+      // quantization on a specific table
+      quantizeTable: (tableName: ERecordKey, callback: (n: any) => any) => {
+        const quantizationService = this.iocContainer.get<IQuantizationService>(
+          IQuantizationServiceType,
+        );
+
+        return quantizationService.quantizeTable(tableName, callback);
+      },
+
+      // keep using temp tables/data to save space
+
+      // another function here -> mapping raw to vectorized data
+
+      // can only be run AFTER a table is quantized, throw error
+      kmeans: (quantizedTable: number[][], k: number) => {
+        const quantizationService = this.iocContainer.get<IQuantizationService>(
+          IQuantizationServiceType,
+        );
+
+        return quantizationService.kmeans(quantizedTable, k);
+      },
+
+      // infer: (modelID: string, userState: string) => {
+      //   const quantizationService = this.iocContainer.get<IQuantizationService>(
+      //     IQuantizationServiceType,
+      //   );
+
+      //   return quantizationService.infer(modelID, userState);
+      // },
+    };
+
     // Questionnaire Methods --------------------------------------------------------------------
     this.questionnaire = {
       getAllQuestionnaires: (

packages/core/tsconfig.json

@@ -40,6 +40,9 @@
     },
     {
       "path": "../signatureVerification"
+    },
+    {
+      "path": "../vector-db"
     }
   ],
   "include": [

packages/docker/Dockerfile

@@ -28,6 +28,7 @@ COPY packages/static-web-integration/package.json /build/packages/static-web-int
 COPY packages/synamint-extension-sdk/package.json /build/packages/synamint-extension-sdk/package.json
 COPY packages/test-harness/package.json /build/packages/test-harness/package.json
 COPY packages/utils/package.json /build/packages/utils/package.json
+COPY packages/vector-db/package.json /build/packages/vector-db/package.json
 COPY packages/web-integration/package.json /build/packages/web-integration/package.json
 COPY packages/web-integration-test/package.json /build/packages/web-integration-test/package.json
 # RUN yarn set version berry

packages/mobile/metro.config.js

@@ -37,6 +37,9 @@ const extraNodeModules = {
   "@snickerdoodlelabs/signature-verification": path.resolve(
     path.join(__dirname, "../signatureVerification"),
   ),
+  "@snickerdoodlelabs/vector-db": path.resolve(
+    path.join(__dirname, "../vector-db"),
+  ),
   stream: require.resolve("stream-browserify"),
   crypto: require.resolve("react-native-quick-crypto"),
   zlib: require.resolve("browserify-zlib"),

packages/objects/src/businessObjects/KMeansResult.ts

@@ -0,0 +1,6 @@
+export interface KMeansResult {
+  clusters: number[];
+  centroids: number[][]; // { centroid: number[], error: number, size: number }[],
+  converged: boolean;
+  iterations: number;
+}

packages/objects/src/businessObjects/index.ts

@@ -22,6 +22,7 @@ export * from "@objects/businessObjects/IndexerSupportSummary.js";
 export * from "@objects/businessObjects/Insight.js";
 export * from "@objects/businessObjects/InsightWithProof.js";
 export * from "@objects/businessObjects/Invitation.js";
+export * from "@objects/businessObjects/KMeansResult.js";
 export * from "@objects/businessObjects/MarketplaceListing.js";
 export * from "@objects/businessObjects/MetatransactionSignatureRequest.js";
 export * from "@objects/businessObjects/NftHoldings.js";

packages/objects/src/interfaces/IIndexedDB.ts

@@ -0,0 +1,118 @@
+import { ResultAsync } from "neverthrow";
+
+import {
+  VersionedObject,
+  VolatileStorageMetadata,
+} from "@objects/businessObjects";
+import { PersistenceError } from "@objects/errors";
+import { IVolatileCursor } from "@objects/interfaces";
+import { VolatileStorageKey } from "@objects/primitives";
+
+export interface IIndexedDB {
+  initialize(): ResultAsync<IDBDatabase, PersistenceError>;
+  close(): ResultAsync<void, PersistenceError>;
+  persist(): ResultAsync<boolean, PersistenceError>;
+  clear(): ResultAsync<void, PersistenceError>;
+  clearObjectStore(name: string): ResultAsync<void, PersistenceError>;
+  putObject<T>(name: string, obj: T): ResultAsync<void, PersistenceError>;
+  removeObject<T extends VersionedObject>(
+    name: string,
+    key: string,
+  ): ResultAsync<VolatileStorageMetadata<T> | null, PersistenceError>;
+
+  getObject<T extends VersionedObject>(
+    name: string,
+    key: VolatileStorageKey,
+    _includeDeleted?: boolean,
+  ): ResultAsync<VolatileStorageMetadata<T> | null, PersistenceError>;
+
+  getCursor<T extends VersionedObject>(
+    name: string,
+    index?: VolatileStorageKey,
+    query?: string | number,
+    direction?: IDBCursorDirection | undefined,
+    mode?: IDBTransactionMode,
+  ): ResultAsync<IVolatileCursor<T>, PersistenceError>;
+
+  deleteDatabase(database: string): ResultAsync<void, PersistenceError>;
+
+  getAll<T extends VersionedObject>(
+    storeName: string,
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getAllByIndex<T extends VersionedObject>(
+    name: string,
+    index: VolatileStorageKey,
+    query: IDBValidKey | IDBKeyRange,
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getAllKeys<T>(
+    name: string,
+    index?: VolatileStorageKey,
+    query?: IDBValidKey | IDBKeyRange,
+    count?: number | undefined,
+  ): ResultAsync<T[], PersistenceError>;
+
+  get<T extends VersionedObject>(
+    name: string,
+    {
+      index,
+      query,
+      count,
+      id,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      count?: number;
+      id?: IDBValidKey;
+    },
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  getKeys(
+    name: string,
+    {
+      index,
+      query,
+      count,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      count?: number;
+    },
+  ): ResultAsync<IDBValidKey[], PersistenceError>;
+
+  getCursor2<T extends VersionedObject>(
+    name: string,
+    {
+      index,
+      query,
+      lowerCount,
+      upperCount,
+      latest,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | null;
+      lowerCount?: number;
+      upperCount?: number;
+      latest?: boolean;
+    },
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+
+  countRecords(
+    name: string,
+    {
+      index,
+      query,
+    }: {
+      index?: string;
+      query?: IDBValidKey | IDBKeyRange | undefined;
+    },
+  ): ResultAsync<number, PersistenceError>;
+
+  getKey<T extends VersionedObject>(
+    tableName: string,
+    obj: VolatileStorageMetadata<T>,
+  ): ResultAsync<VolatileStorageKey | null, PersistenceError>;
+}
+
+export const IIndexedDBType = Symbol.for("IIndexedDB");

packages/objects/src/interfaces/ISnickerdoodleCore.ts

@@ -42,13 +42,17 @@ import {
   QuestionnaireWithAnswers,
   QuestionnaireAnswer,
   NewQuestionnaireAnswer,
+  KMeansResult,
+  VersionedObject,
+  VolatileStorageMetadata,
   // AuthenticatedStorageParams,
 } from "@objects/businessObjects/index.js";
 import {
   EChain,
   ECloudStorageType,
   EDataWalletPermission,
   EInvitationStatus,
+  ERecordKey,
 } from "@objects/enum/index.js";
 import {
   AccountIndexingError,
@@ -83,6 +87,7 @@ import {
   MethodSupportError,
 } from "@objects/errors/index.js";
 import { IConsentCapacity } from "@objects/interfaces/IConsentCapacity.js";
+import { IIndexedDB } from "@objects/interfaces/IIndexedDB.js";
 import { IOldUserAgreement } from "@objects/interfaces/IOldUserAgreement.js";
 import { ISnickerdoodleCoreEvents } from "@objects/interfaces/ISnickerdoodleCoreEvents.js";
 import { IUserAgreement } from "@objects/interfaces/IUserAgreement.js";
@@ -716,6 +721,25 @@ export interface IStorageMethods {
   ): ResultAsync<URLString, never>;
 }
 
+export interface IVectorQuantizationMethods {
+  initialize(template?: IIndexedDB): ResultAsync<IIndexedDB, PersistenceError>;
+  table<T extends VersionedObject>(
+    name: string,
+  ): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+  quantizeTable(
+    tableName: ERecordKey,
+    callback: (n: any) => any,
+  ): ResultAsync<number[][], PersistenceError>;
+  kmeans(
+    quantizedTable: number[][],
+    k: number,
+  ): ResultAsync<KMeansResult, PersistenceError>;
+  // infer(
+  //   model: KMeansResult,
+  //   userState: number[][],
+  // ): ResultAsync<number[][], PersistenceError>;
+}
+
 export interface IQuestionnaireMethods {
   /**
    * Returns a list of questionnaires that the user can complete (that do not already have answers),

packages/objects/src/interfaces/IVolatileCursor.ts

@@ -0,0 +1,15 @@
+import { ResultAsync } from "neverthrow";
+
+import {
+  VolatileStorageMetadata,
+  VersionedObject,
+} from "@objects/businessObjects";
+import { PersistenceError } from "@objects/errors";
+
+export interface IVolatileCursor<T extends VersionedObject> {
+  nextValue(): ResultAsync<T | null, PersistenceError>;
+  allValues(): ResultAsync<T[] | null, PersistenceError>;
+
+  _next(): ResultAsync<VolatileStorageMetadata<T> | null, PersistenceError>;
+  _all(): ResultAsync<VolatileStorageMetadata<T>[], PersistenceError>;
+}