Blocking: build hash schema without spark-mllib SchemaUtils (#1418)

* Blocking: build hash schema without spark-mllib SchemaUtils

SparkBlockingTreeUtil.appendHashCol used org.apache.spark.ml.util.SchemaUtils
just to append the z_hash column to a schema. That pulls spark-mllib into
blocking, and spark-mllib is not on a Spark Connect client classpath, so
getBlockHashes failed there with ClassNotFoundException.

Use StructType.add with the same duplicate column check SchemaUtils had, so
behaviour on classic Spark is unchanged. Adds a unit test for appendHashCol.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Blocking: drop duplicate z_hash column check in appendHashCol

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: sonalgoyal

spark/core/src/main/java/zingg/spark/core/util/SparkBlockingTreeUtil.java

@@ -5,7 +5,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.spark.ml.util.SchemaUtils;
 import org.apache.spark.sql.Column;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
@@ -47,7 +46,9 @@ public ZFrame<Dataset<Row>, Row, Column> getBlockHashes(ZFrame<Dataset<Row>, Row
     }
 
     public StructType appendHashCol(StructType s) {
-        StructType retSchema = SchemaUtils.appendColumn(s, ColName.HASH_COL, DataTypes.IntegerType, false);
+        // plain StructType instead of spark ml SchemaUtils.appendColumn, so blocking does not need
+        // spark-mllib on the classpath (it is not there for Spark Connect clients)
+        StructType retSchema = s.add(ColName.HASH_COL, DataTypes.IntegerType, false);
         LOG.debug("returning schema after step 1 is " + retSchema);
         return retSchema;
 }

spark/core/src/test/java/zingg/spark/core/util/TestSparkBlockingTreeUtilSchema.java

@@ -0,0 +1,39 @@
+package zingg.spark.core.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+import zingg.common.client.util.ColName;
+
+public class TestSparkBlockingTreeUtilSchema {
+
+	private final SparkBlockingTreeUtil blockingTreeUtil = new SparkBlockingTreeUtil(null, null);
+
+	private StructType getSchema() {
+		return DataTypes.createStructType(new StructField[] {
+			DataTypes.createStructField("id", DataTypes.StringType, true),
+			DataTypes.createStructField("fname", DataTypes.StringType, true)
+		});
+	}
+
+	@Test
+	public void testAppendHashColAddsNonNullableIntAtEnd() {
+		StructType schema = getSchema();
+		StructType withHash = blockingTreeUtil.appendHashCol(schema);
+
+		assertEquals(3, withHash.fields().length);
+		assertEquals("id", withHash.fields()[0].name());
+		assertEquals("fname", withHash.fields()[1].name());
+		StructField hash = withHash.fields()[2];
+		assertEquals(ColName.HASH_COL, hash.name());
+		assertEquals(DataTypes.IntegerType, hash.dataType());
+		assertFalse(hash.nullable());
+		// input schema is left unchanged
+		assertEquals(2, schema.fields().length);
+	}
+}