diff --git a/auron-core/pom.xml b/auron-core/pom.xml
index 414553e31..8565aeb7e 100644
--- a/auron-core/pom.xml
+++ b/auron-core/pom.xml
@@ -77,6 +77,12 @@
provided
+
+ org.apache.hadoop
+ hadoop-client-runtime
+ test
+
+
org.junit.jupiter
junit-jupiter-api
diff --git a/auron-core/src/main/java/org/apache/auron/jni/JniBridge.java b/auron-core/src/main/java/org/apache/auron/jni/JniBridge.java
index 48f10ab01..1f145be3f 100644
--- a/auron-core/src/main/java/org/apache/auron/jni/JniBridge.java
+++ b/auron-core/src/main/java/org/apache/auron/jni/JniBridge.java
@@ -19,7 +19,6 @@
import java.io.IOException;
import java.lang.management.BufferPoolMXBean;
import java.lang.management.ManagementFactory;
-import java.net.URI;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@@ -71,12 +70,11 @@ public static void putResource(String key, Object value) {
}
public static FSDataInputWrapper openFileAsDataInputWrapper(FileSystem fs, String path) throws Exception {
- // the path is a URI string, so we need to convert it to a URI object
- return FSDataInputWrapper.wrap(fs.open(new Path(new URI(path))));
+ return FSDataInputWrapper.wrap(fs.open(new Path(path)));
}
public static FSDataOutputWrapper createFileAsDataOutputWrapper(FileSystem fs, String path) throws Exception {
- return FSDataOutputWrapper.wrap(fs.create(new Path(new URI(path))));
+ return FSDataOutputWrapper.wrap(fs.create(new Path(path)));
}
public static long getDirectMemoryUsed() {
diff --git a/auron-core/src/test/java/org/apache/auron/jni/JniBridgeTest.java b/auron-core/src/test/java/org/apache/auron/jni/JniBridgeTest.java
new file mode 100644
index 000000000..05489643d
--- /dev/null
+++ b/auron-core/src/test/java/org/apache/auron/jni/JniBridgeTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.jni;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.jupiter.api.Test;
+
+public class JniBridgeTest {
+
+ @Test
+ public void testFileWrappersPreserveLiteralHashInHdfsPath() throws Exception {
+ String path = "hdfs://mycluster/tmp/channel=wx_repro#mini/spark-submit-123/part-00000.json";
+ CapturingFileSystem cfs = new CapturingFileSystem();
+
+ JniBridge.openFileAsDataInputWrapper(cfs, path).close();
+ JniBridge.createFileAsDataOutputWrapper(cfs, path).close();
+
+ assertPathPreservesHash(cfs.openedPath);
+ assertPathPreservesHash(cfs.createdPath);
+ }
+
+ @Test
+ public void testFileWrappersPreserveNormalizedPercentPathStrings() throws Exception {
+ String path = "file:/tmp/t1/part=test%20test/part-00000.parquet";
+ CapturingFileSystem cfs = new CapturingFileSystem();
+
+ JniBridge.openFileAsDataInputWrapper(cfs, path).close();
+ JniBridge.createFileAsDataOutputWrapper(cfs, path).close();
+
+ assertEquals(
+ "/tmp/t1/part=test%20test/part-00000.parquet",
+ cfs.openedPath.toUri().getPath());
+ assertEquals(
+ "/tmp/t1/part=test%20test/part-00000.parquet",
+ cfs.createdPath.toUri().getPath());
+ }
+
+ @Test
+ public void testHadoopPathUriAcceptsFilePathWithSpace() throws Exception {
+ String path = "file:/tmp/t1/part=test test/part-00000.parquet";
+
+ assertThrows(URISyntaxException.class, () -> new URI(path));
+ assertEquals("file", new Path(path).toUri().getScheme());
+ assertEquals(
+ "/tmp/t1/part=test test/part-00000.parquet",
+ new Path(path).toUri().getPath());
+ }
+
+ private static void assertPathPreservesHash(Path path) {
+ assertEquals(
+ "/tmp/channel=wx_repro#mini/spark-submit-123/part-00000.json",
+ path.toUri().getPath());
+ assertNull(path.toUri().getFragment());
+ }
+
+ private static class CapturingFileSystem extends RawLocalFileSystem {
+ private final Statistics statistics = new Statistics("hdfs");
+ private Path openedPath;
+ private Path createdPath;
+
+ @Override
+ public FSDataInputStream open(Path path) throws IOException {
+ openedPath = path;
+ return new FSDataInputStream(new EmptyFSInputStream());
+ }
+
+ @Override
+ public FSDataOutputStream create(Path path) throws IOException {
+ createdPath = path;
+ return new FSDataOutputStream(new ByteArrayOutputStream(), statistics);
+ }
+ }
+
+ private static class EmptyFSInputStream extends FSInputStream {
+ @Override
+ public void seek(long pos) {}
+
+ @Override
+ public long getPos() {
+ return 0;
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) {
+ return false;
+ }
+
+ @Override
+ public int read() {
+ return -1;
+ }
+ }
+}
diff --git a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
index 0e8a2c8e7..607882e0b 100644
--- a/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
+++ b/spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
@@ -971,6 +971,18 @@ class ShimsImpl extends Shims with Logging {
size: Long): PartitionedFile =
PartitionedFile(partitionValues, filePath, offset, size)
+ @sparkver("3.0 / 3.1 / 3.2 / 3.3")
+ override def getPartitionedFilePathString(file: PartitionedFile): String = {
+ import org.apache.hadoop.fs.Path
+ import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
+
+ // Spark 3.0-3.3 PartitionedFile.filePath is URI-encoded.
+ file.filePath
+ .split(Path.SEPARATOR, -1)
+ .map(ExternalCatalogUtils.unescapePathName)
+ .mkString(Path.SEPARATOR)
+ }
+
@sparkver("3.4 / 3.5 / 4.0 / 4.1")
override def getPartitionedFile(
partitionValues: InternalRow,
@@ -982,6 +994,10 @@ class ShimsImpl extends Shims with Logging {
PartitionedFile(partitionValues, SparkPath.fromPath(new Path(filePath)), offset, size)
}
+ @sparkver("3.4 / 3.5 / 4.0 / 4.1")
+ override def getPartitionedFilePathString(file: PartitionedFile): String =
+ file.toPath.toString
+
@sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1")
override def getMinPartitionNum(sparkSession: SparkSession): Int =
sparkSession.sessionState.conf.filesMinPartitionNum
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
index 19f98b415..896a8d4cd 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala
@@ -272,6 +272,8 @@ abstract class Shims {
offset: Long,
size: Long): PartitionedFile
+ def getPartitionedFilePathString(file: PartitionedFile): String
+
def getMinPartitionNum(sparkSession: SparkSession): Int
@nowarn("cat=unused") // Some params temporarily unused
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeFileSourceScanBase.scala b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeFileSourceScanBase.scala
index 84e886060..bf038184a 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeFileSourceScanBase.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeFileSourceScanBase.scala
@@ -16,13 +16,12 @@
*/
package org.apache.spark.sql.execution.auron.plan
-import java.net.URI
import java.security.PrivilegedExceptionAction
import scala.jdk.CollectionConverters._
import org.apache.commons.lang3.reflect.MethodUtils
-import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.MapPartitionsRDD
import org.apache.spark.sql.auron.NativeConverters
@@ -114,7 +113,7 @@ abstract class NativeFileSourceScanBase(basedFileScan: FileSourceScanExec)
}
pb.PartitionedFile
.newBuilder()
- .setPath(s"${file.filePath}")
+ .setPath(Shims.get.getPartitionedFilePathString(file))
.setSize(fileSizes(file.filePath))
.addAllPartitionValues(nativePartitionValues.asJava)
.setLastModifiedNs(0)
@@ -149,7 +148,7 @@ abstract class NativeFileSourceScanBase(basedFileScan: FileSourceScanExec)
val currentTimeMillis = System.currentTimeMillis()
val fs = NativeHelper.currentUser.doAs(new PrivilegedExceptionAction[FileSystem] {
override def run(): FileSystem = {
- FileSystem.get(new URI(location), sharedConf)
+ FileSystem.get(new Path(location).toUri, sharedConf)
}
})
getFsTimeMetric.add((System.currentTimeMillis() - currentTimeMillis) * 1000000)