Market scan

docs/FEATURES.md

@@ -13,7 +13,8 @@
   - [Custom filters](#custom-filters)
   - [Custom ranking of proposals](#custom-ranking-of-proposals)
   - [Uploading local images to the provider](#uploading-local-images-to-the-provider)
-    - [Setup and teardown methods](#setup-and-teardown-methods)
+  - [Setup and teardown methods](#setup-and-teardown-methods)
+  - [Market scan](#market-scan)
   - [Read more](#read-more)
   <!-- TOC -->
 
@@ -151,7 +152,7 @@ const order: MarketOrderSpec = {
 
 [Check the full example](../examples/advanced//local-image/)
 
-### Setup and teardown methods
+## Setup and teardown methods
 
 You can define a setup method that will be executed the first time a provider is rented and a teardown method
 that will be executed before the rental is done. This is useful when you want to avoid doing the same work
@@ -174,26 +175,26 @@ const pool = await glm.manyOf({
 
 [Check the full example](../examples/advanced/setup-and-teardown.ts)
 
-<!--
-TODO:
- ### Market scan
+## Market scan
 
 You can scan the market for available providers and their offers. This is useful when you want to see what's available
 before placing an order.
 
 ```ts
-await glm.market.scan(order).subscribe({
-  next: (proposal) => {
-    console.log("Received proposal from provider", proposal.provider.name);
-  },
-  complete: () => {
-    console.log("Market scan completed");
-  },
-});
+await glm.market
+  .scan(order)
+  .pipe(takeUntil(timer(10_000)))
+  .subscribe({
+    next: (scannedOffer) => {
+      console.log("Found offer from", scannedOffer.provider.name);
+    },
+    complete: () => {
+      console.log("Market scan completed");
+    },
+  });
 ```
 
-[Check the full example](../examples/basic/market-scan.ts)
--->
+[Check the full example](../examples/advanced/scan.ts)
 
 ## Read more
 

examples/advanced/scan.ts

@@ -0,0 +1,65 @@
+/**
+ * This example demonstrates how to scan the market for providers that meet specific requirements.
+ */
+import { GolemNetwork, ScanOptions } from "@golem-sdk/golem-js";
+import { last, map, scan, takeUntil, tap, timer } from "rxjs";
+
+// What providers are we looking for?
+const scanOptions: ScanOptions = {
+  // fairly powerful machine but not too powerful
+  workload: {
+    engine: "vm",
+    minCpuCores: 4,
+    maxCpuCores: 16,
+    minMemGib: 4,
+    maxMemGib: 8,
+    capabilities: ["vpn"],
+    minStorageGib: 16,
+  },
+  // let's look at mainnet providers only
+  payment: {
+    network: "polygon",
+  },
+};
+
+(async () => {
+  const glm = new GolemNetwork();
+  await glm.connect();
+  const spec = glm.market.buildScanSpecification(scanOptions);
+
+  // For advanced users: you can also add constraints manually:
+  // spec.constraints.push("(golem.node.id.name=my-favorite-provider)");
+
+  const SCAN_DURATION_MS = 10_000;
+
+  console.log(`Scanning for ${SCAN_DURATION_MS / 1000} seconds...`);
+  glm.market
+    .scan(spec)
+    .pipe(
+      tap((scannedOffer) => {
+        console.log("Found offer from", scannedOffer.provider.name);
+      }),
+      // calculate the cost of an hour of work
+      map(
+        (scannedOffer) =>
+          scannedOffer.pricing.start + //
+          scannedOffer.pricing.cpuSec * 3600 +
+          scannedOffer.pricing.envSec * 3600,
+      ),
+      // calculate the running average
+      scan((total, cost) => total + cost, 0),
+      map((totalCost, index) => totalCost / (index + 1)),
+      // stop scanning after SCAN_DURATION_MS
+      takeUntil(timer(SCAN_DURATION_MS)),
+      last(),
+    )
+    .subscribe({
+      next: (averageCost) => {
+        console.log("Average cost for an hour of work:", averageCost.toFixed(6), "GLM");
+      },
+      complete: () => {
+        console.log("Scan completed, shutting down...");
+        glm.disconnect();
+      },
+    });
+})();

examples/package.json

@@ -17,6 +17,7 @@
     "advanced-payment-filters": "tsx advanced/payment-filters.ts",
     "advanced-proposal-filters": "tsx advanced/proposal-filter.ts",
     "advanced-proposal-predefined-filter": "tsx advanced/proposal-predefined-filter.ts",
+    "advanced-scan": "tsx advanced/scan.ts",
     "advanced-setup-and-teardown": "tsx advanced/setup-and-teardown.ts",
     "local-image": "tsx advanced/local-image/serveLocalGvmi.ts",
     "deployment": "tsx experimental/deployment/new-api.ts",

package.json

@@ -77,7 +77,7 @@
     "tmp": "^0.2.2",
     "uuid": "^10.0.0",
     "ws": "^8.16.0",
-    "ya-ts-client": "^1.1.2"
+    "ya-ts-client": "^1.1.3"
   },
   "devDependencies": {
     "@commitlint/cli": "^19.0.3",

src/market/api.ts

@@ -3,6 +3,7 @@ import { Demand, DemandBodyPrototype, DemandSpecification } from "./demand";
 import { MarketProposalEvent, OfferCounterProposal, OfferProposal } from "./proposal";
 import { Agreement, AgreementEvent, AgreementState } from "./agreement";
 import { AgreementOptions } from "./agreement/agreement";
+import { ScanSpecification, ScannedOffer } from "./scan";
 
 export type MarketEvents = {
   demandSubscriptionStarted: (event: { demand: Demand }) => void;
@@ -132,4 +133,9 @@ export interface IMarketApi {
    * Retrieves the state of an agreement based on the provided agreement ID.
    */
   getAgreementState(id: string): Promise<AgreementState>;
+
+  /**
+   * Scan the market for offers that match the given specification.
+   */
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer>;
 }

src/market/error.ts

@@ -15,6 +15,7 @@ export enum MarketErrorCode {
   AgreementApprovalFailed = "AgreementApprovalFailed",
   NoProposalAvailable = "NoProposalAvailable",
   InternalError = "InternalError",
+  ScanFailed = "ScanFailed",
 }
 
 export class GolemMarketError extends GolemModuleError {

src/market/index.ts

@@ -12,3 +12,4 @@ export { BasicDemandDirector } from "./demand/directors/basic-demand-director";
 export { PaymentDemandDirector } from "./demand/directors/payment-demand-director";
 export { WorkloadDemandDirector } from "./demand/directors/workload-demand-director";
 export * from "./proposal/market-proposal-event";
+export * from "./scan";

src/market/market.module.ts

@@ -40,6 +40,7 @@ import { GolemAbortError, GolemTimeoutError, GolemUserError } from "../shared/er
 import { MarketOrderSpec } from "../golem-network";
 import { INetworkApi, NetworkModule } from "../network";
 import { AgreementOptions } from "./agreement/agreement";
+import { ScanDirector, ScanOptions, ScanSpecification, ScannedOffer } from "./scan";
 
 export type DemandEngine = "vm" | "vm-nvidia" | "wasmtime";
 
@@ -94,6 +95,14 @@ export interface MarketModule {
     allocation: Allocation,
   ): Promise<DemandSpecification>;
 
+  /**
+   * Build a ScanSpecification that can be used to scan the market for offers.
+   * The difference between this method and `buildDemandDetails` is that this method does not require an
+   * allocation, doesn't inherit payment properties from `GolemNetwork` settings and doesn't provide any defaults.
+   * If you wish to set the payment platform, you need to specify it in the ScanOptions.
+   */
+  buildScanSpecification(options: ScanOptions): ScanSpecification;
+
   /**
    * Publishes the demand to the market and handles refreshing it when needed.
    * Each time the demand is refreshed, a new demand is emitted by the observable.
@@ -202,6 +211,11 @@ export interface MarketModule {
    * Fetch the most up-to-date agreement details from the yagna
    */
   fetchAgreement(agreementId: string): Promise<Agreement>;
+
+  /**
+   * Scan the market for offers that match the given demand specification.
+   */
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer>;
 }
 
 /**
@@ -285,6 +299,13 @@ export class MarketModuleImpl implements MarketModule {
     return new DemandSpecification(builder.getProduct(), allocation.paymentPlatform);
   }
 
+  buildScanSpecification(options: ScanOptions): ScanSpecification {
+    const builder = new DemandBodyBuilder();
+    const director = new ScanDirector(options);
+    director.apply(builder);
+    return builder.getProduct();
+  }
+
   /**
    * Augments the user-provided options with additional logic
    *
@@ -707,4 +728,8 @@ export class MarketModuleImpl implements MarketModule {
     }
     return isPriceValid;
   }
+
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer> {
+    return this.deps.marketApi.scan(scanSpecification);
+  }
 }

src/market/scan/index.ts

@@ -0,0 +1,3 @@
+export * from "./types";
+export * from "./scan-director";
+export * from "./scanned-proposal";

src/market/scan/scan-director.ts

@@ -0,0 +1,59 @@
+import { ComparisonOperator, DemandBodyBuilder } from "../demand";
+import { ScanOptions } from "./types";
+
+export class ScanDirector {
+  constructor(private options: ScanOptions) {}
+
+  public async apply(builder: DemandBodyBuilder) {
+    this.addWorkloadDecorations(builder);
+    this.addGenericDecorations(builder);
+    this.addPaymentDecorations(builder);
+  }
+
+  private addPaymentDecorations(builder: DemandBodyBuilder): void {
+    if (!this.options.payment) return;
+    const network = this.options.payment.network;
+    const driver = this.options.payment.driver || "erc20";
+    const token = this.options.payment.token || ["mainnet", "polygon"].includes(network) ? "glm" : "tglm";
+    builder.addConstraint(`golem.com.payment.platform.${driver}-${network}-${token}.address`, "*");
+  }
+
+  private addWorkloadDecorations(builder: DemandBodyBuilder): void {
+    if (this.options.workload?.engine) {
+      builder.addConstraint("golem.runtime.name", this.options.workload?.engine);
+    }
+    if (this.options.workload?.capabilities)
+      this.options.workload?.capabilities.forEach((cap) => builder.addConstraint("golem.runtime.capabilities", cap));
+
+    if (this.options.workload?.minMemGib) {
+      builder.addConstraint("golem.inf.mem.gib", this.options.workload?.minMemGib, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.maxMemGib) {
+      builder.addConstraint("golem.inf.mem.gib", this.options.workload?.maxMemGib, ComparisonOperator.LtEq);
+    }
+    if (this.options.workload?.minStorageGib) {
+      builder.addConstraint("golem.inf.storage.gib", this.options.workload?.minStorageGib, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.maxStorageGib) {
+      builder.addConstraint("golem.inf.storage.gib", this.options.workload?.maxStorageGib, ComparisonOperator.LtEq);
+    }
+    if (this.options.workload?.minCpuThreads) {
+      builder.addConstraint("golem.inf.cpu.threads", this.options.workload?.minCpuThreads, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.maxCpuThreads) {
+      builder.addConstraint("golem.inf.cpu.threads", this.options.workload?.maxCpuThreads, ComparisonOperator.LtEq);
+    }
+    if (this.options.workload?.minCpuCores) {
+      builder.addConstraint("golem.inf.cpu.cores", this.options.workload?.minCpuCores, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.maxCpuCores) {
+      builder.addConstraint("golem.inf.cpu.cores", this.options.workload?.maxCpuCores, ComparisonOperator.LtEq);
+    }
+  }
+
+  private addGenericDecorations(builder: DemandBodyBuilder): void {
+    if (this.options.subnetTag) {
+      builder.addConstraint("golem.node.debug.subnet", this.options.subnetTag);
+    }
+  }
+}