CI TESTING - DO NOT MERGE!

.github/workflows/maven-checks.yml

@@ -24,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         java: [ 8.0.442, 17.0.13 ]
-    runs-on: ubuntu-latest
+    runs-on: yscope-gh-runner
     timeout-minutes: 45
     steps:
       - name: Free Disk Space
@@ -68,7 +68,7 @@ jobs:
   presto-coordinator-image:
     name: "presto-coordinator-image"
     needs: "maven-checks"
-    runs-on: "ubuntu-22.04"
+    runs-on: "yscope-gh-runner"
     steps:
       - uses: "actions/checkout@v4"
         with:

.github/workflows/prestissimo-worker-images-build.yml

@@ -7,43 +7,78 @@ on:
 jobs:
   prestissimo-worker-images-build:
     name: "prestissimo-worker-images-build"
-    runs-on: "ubuntu-22.04"
+    runs-on: "yscope-gh-runner"
     steps:
       - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
         with:
           submodules: "recursive"
 
+      - name: "Set up Docker Buildx"
+        uses: "docker/setup-buildx-action@v3"
+
       - name: "Login to image registry"
         uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
         with:
           registry: "ghcr.io"
           username: "${{github.actor}}"
           password: "${{secrets.GITHUB_TOKEN}}"
 
+      - name: "Calculate dependency hash"
+        id: "deps-hash"
+        run: |-
+          # Hash the dockerfile, setup scripts, and velox submodule commit
+          VELOX_COMMIT=$(git -C presto-native-execution/velox rev-parse HEAD)
+          HASH=$(cat \
+            presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile \
+            presto-native-execution/scripts/setup-ubuntu.sh \
+            presto-native-execution/scripts/setup-adapters.sh \
+            presto-native-execution/velox/scripts/setup-ubuntu.sh \
+            <(echo "velox:${VELOX_COMMIT}") \
+            | sha256sum | cut -d' ' -f1 | cut -c1-12)
+          echo "hash=${HASH}" >> $GITHUB_OUTPUT
+          echo "Dependency hash: ${HASH}"
+          echo "dep_image=ghcr.io/${{github.repository}}/prestissimo-worker-dev-env:${HASH}" >> $GITHUB_OUTPUT
+
+      - name: "Check if dependency image exists"
+        id: "check-deps-image"
+        run: |-
+          # Try to pull the image with the hash tag
+          if docker pull ghcr.io/${{github.repository}}/prestissimo-worker-dev-env:${{steps.deps-hash.outputs.hash}} 2>/dev/null; then
+            echo "exists=true" >> $GITHUB_OUTPUT
+            echo "✓ Dependency image found in cache, skipping 1+ hour build"
+          else
+            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "✗ Dependency image not found, will build from scratch"
+          fi
+
       - name: "Set up metadata for dependency image"
         id: "metadata-deps-image"
         uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
         with:
           images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env"
-          tags: "type=raw,value=dev"
+          tags: |-
+            type=raw,value=test
+            type=raw,value=${{steps.deps-hash.outputs.hash}}
 
       - name: "Build and push dependency image"
+        if: steps.check-deps-image.outputs.exists != 'true'
         uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
         with:
           context: "./presto-native-execution"
           file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile"
-          push: >-
-            ${{github.event_name != 'pull_request'
-            && github.ref == 'refs/heads/release-0.293-clp-connector'}}
+          push: "${{github.event_name != 'pull_request'}}"
           tags: "${{steps.metadata-deps-image.outputs.tags}}"
           labels: "${{steps.metadata-deps-image.outputs.labels}}"
+          load: "${{github.event_name == 'pull_request'}}"
+          cache-from: "type=registry,ref=ghcr.io/${{github.repository}}/prestissimo-worker-dev-env:buildcache"
+          cache-to: "type=registry,ref=ghcr.io/${{github.repository}}/prestissimo-worker-dev-env:buildcache,mode=max"
 
       - name: "Set up metadata for runtime image"
         id: "metadata-runtime-image"
         uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
         with:
           images: "ghcr.io/${{github.repository}}/prestissimo-worker"
-          tags: "type=raw,value=dev"
+          tags: "type=raw,value=test"
 
       - name: "Get number of cores"
         id: "get-cores"
@@ -55,16 +90,14 @@ jobs:
         with:
           build-args: |-
             BASE_IMAGE=ubuntu:22.04
-            DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}}
+            DEPENDENCY_IMAGE=${{steps.deps-hash.outputs.dep_image}}
             EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \
               -DPRESTO_ENABLE_PARQUET=ON \
               -DPRESTO_ENABLE_S3=ON
             NUM_THREADS=${{steps.get-cores.outputs.num_cores}}
             OSNAME=ubuntu
           context: "./presto-native-execution"
           file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile"
-          push: >-
-            ${{github.event_name != 'pull_request'
-            && github.ref == 'refs/heads/release-0.293-clp-connector'}}
+          push: "${{github.event_name != 'pull_request'}}"
           tags: "${{steps.metadata-runtime-image.outputs.tags}}"
           labels: "${{steps.metadata-runtime-image.outputs.labels}}"

.github/workflows/prestocpp-format-and-header-check.yml

@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   prestocpp-format-and-header-check:
-    runs-on: ubuntu-latest
+    runs-on: yscope-gh-runner
     container:
       image: public.ecr.aws/oss-presto/velox-dev:check
     steps:

.github/workflows/prestocpp-linux-build-and-unit-test.yml

@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   prestocpp-linux-build-for-test:
-    runs-on: ubuntu-22.04
+    runs-on: yscope-gh-runner
     container:
       image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
     env:
@@ -97,7 +97,7 @@ jobs:
 
   prestocpp-linux-presto-e2e-tests:
     needs: prestocpp-linux-build-for-test
-    runs-on: ubuntu-22.04
+    runs-on: yscope-gh-runner
     container:
       image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
     env:
@@ -172,7 +172,7 @@ jobs:
 
   prestocpp-linux-presto-native-tests:
     needs: prestocpp-linux-build-for-test
-    runs-on: ubuntu-22.04
+    runs-on: yscope-gh-runner
     strategy:
       fail-fast: false
       matrix:
@@ -248,7 +248,7 @@ jobs:
 
   prestocpp-linux-presto-sidecar-tests:
     needs: prestocpp-linux-build-for-test
-    runs-on: ubuntu-22.04
+    runs-on: yscope-gh-runner
     container:
       image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
     env:

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   changes:
-    runs-on: ubuntu-latest
+    runs-on: yscope-gh-runner
     # Required permissions
     permissions:
       pull-requests: read
@@ -41,7 +41,7 @@ jobs:
             - '!presto-docs/**'
 
   test:
-    runs-on: ubuntu-latest
+    runs-on: yscope-gh-runner
     needs: changes
     strategy:
       fail-fast: false

presto-clp/pom.xml

@@ -75,6 +75,11 @@
             <artifactId>jackson-databind</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.facebook.presto</groupId>
             <artifactId>presto-spi</artifactId>
@@ -154,4 +159,14 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>2.1</version> <!-- upper bound -->
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
 </project>

presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpConfig.java

@@ -33,6 +33,8 @@ public class ClpConfig
     private long metadataRefreshInterval = 60;
     private long metadataExpireInterval = 600;
 
+    private String metadataYamlPath;
+
     private String splitFilterConfig;
     private SplitFilterProviderType splitFilterProviderType = SplitFilterProviderType.MYSQL;
     private SplitProviderType splitProviderType = SplitProviderType.MYSQL;
@@ -151,6 +153,18 @@ public ClpConfig setMetadataExpireInterval(long metadataExpireInterval)
         return this;
     }
 
+    public String getMetadataYamlPath()
+    {
+        return metadataYamlPath;
+    }
+
+    @Config("clp.metadata-yaml-path")
+    public ClpConfig setMetadataYamlPath(String metadataYamlPath)
+    {
+        this.metadataYamlPath = metadataYamlPath;
+        return this;
+    }
+
     public String getSplitFilterConfig()
     {
         return splitFilterConfig;
@@ -189,7 +203,8 @@ public ClpConfig setSplitProviderType(SplitProviderType splitProviderType)
 
     public enum MetadataProviderType
     {
-        MYSQL
+        MYSQL,
+        YAML
     }
 
     public enum SplitFilterProviderType
@@ -199,6 +214,7 @@ public enum SplitFilterProviderType
 
     public enum SplitProviderType
     {
-        MYSQL
+        MYSQL,
+        PINOT
     }
 }

presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpErrorCode.java

@@ -28,6 +28,7 @@ public enum ClpErrorCode
     CLP_UNSUPPORTED_SPLIT_SOURCE(2, EXTERNAL),
     CLP_UNSUPPORTED_TYPE(3, EXTERNAL),
     CLP_UNSUPPORTED_CONFIG_OPTION(4, EXTERNAL),
+    CLP_UNSUPPORTED_TABLE_SCHEMA_YAML(5, EXTERNAL),
 
     CLP_SPLIT_FILTER_CONFIG_NOT_FOUND(10, USER_ERROR),
     CLP_MANDATORY_SPLIT_FILTER_NOT_VALID(11, USER_ERROR),

presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpModule.java

@@ -16,7 +16,9 @@
 import com.facebook.airlift.configuration.AbstractConfigurationAwareModule;
 import com.facebook.presto.plugin.clp.metadata.ClpMetadataProvider;
 import com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider;
+import com.facebook.presto.plugin.clp.metadata.ClpYamlMetadataProvider;
 import com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider;
+import com.facebook.presto.plugin.clp.split.ClpPinotSplitProvider;
 import com.facebook.presto.plugin.clp.split.ClpSplitProvider;
 import com.facebook.presto.plugin.clp.split.filter.ClpMySqlSplitFilterProvider;
 import com.facebook.presto.plugin.clp.split.filter.ClpSplitFilterProvider;
@@ -56,13 +58,19 @@ protected void setup(Binder binder)
         if (config.getMetadataProviderType() == MetadataProviderType.MYSQL) {
             binder.bind(ClpMetadataProvider.class).to(ClpMySqlMetadataProvider.class).in(Scopes.SINGLETON);
         }
+        else if (config.getMetadataProviderType() == MetadataProviderType.YAML) {
+            binder.bind(ClpMetadataProvider.class).to(ClpYamlMetadataProvider.class).in(Scopes.SINGLETON);
+        }
         else {
             throw new PrestoException(CLP_UNSUPPORTED_METADATA_SOURCE, "Unsupported metadata provider type: " + config.getMetadataProviderType());
         }
 
         if (config.getSplitProviderType() == SplitProviderType.MYSQL) {
             binder.bind(ClpSplitProvider.class).to(ClpMySqlSplitProvider.class).in(Scopes.SINGLETON);
         }
+        else if (config.getSplitProviderType() == SplitProviderType.PINOT) {
+            binder.bind(ClpSplitProvider.class).to(ClpPinotSplitProvider.class).in(Scopes.SINGLETON);
+        }
         else {
             throw new PrestoException(CLP_UNSUPPORTED_SPLIT_SOURCE, "Unsupported split provider type: " + config.getSplitProviderType());
         }

presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpSplit.java

@@ -24,6 +24,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
 import static com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE;
@@ -77,6 +78,25 @@ public List<HostAddress> getPreferredNodes(NodeProvider nodeProvider)
         return ImmutableList.of();
     }
 
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(path, type, kqlQuery);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        ClpSplit other = (ClpSplit) obj;
+        return this.type == other.type && this.path.equals(other.path) && this.kqlQuery.equals(other.kqlQuery);
+    }
+
     @Override
     public Map<String, String> getInfo()
     {