GH-3558: Properly close buffers

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/CodecFactory.java

@@ -385,7 +385,12 @@ private CompressionCodec getCodecAtLevel(CompressionCodecName codecName, int lev
         break;
       case GZIP:
         validateGzipLevel(level);
-        levelConf.setEnum(GZIP_COMPRESS_LEVEL, zlibCompressionLevel(level));
+        // Store the enum constant name rather than using Configuration#setEnum, which persists
+        // Enum#toString(). Hadoop reads this back via Configuration#getEnum -> Enum#valueOf, which
+        // requires the constant name. In some Hadoop builds ZlibCompressor.CompressionLevel
+        // overrides toString() to return the numeric level (e.g. "5"), so setEnum would write a
+        // value that valueOf cannot resolve, throwing "No enum constant ...CompressionLevel.5".
+        levelConf.set(GZIP_COMPRESS_LEVEL, zlibCompressionLevel(level).name());
         break;
       case BROTLI:
         validateBrotliLevel(level);
@@ -464,7 +469,12 @@ private String cacheKey(CompressionCodecName codecName) {
 
   private String cacheKey(CompressionCodecName codecName, int level) {
     String codecClass = codecName.getHadoopCompressionCodecClassName();
-    return (codecClass == null ? codecName.name() : codecClass) + ":" + level;
+    // Use a distinct namespace ("#level=") for the leveled path so that this key can never
+    // collide with the no-level cacheKey(codecName), which appends the raw configuration value
+    // (e.g. "<codecClass>:5" when zlib.compress.level=5 is set directly). Both maps that use
+    // these keys (the per-factory compressors map and the shared static CODEC_BY_NAME) would
+    // otherwise return a codec built from the raw config instead of the level-configured one.
+    return (codecClass == null ? codecName.name() : codecClass) + "#level=" + level;
   }
 
   @Override

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java

@@ -1190,10 +1190,18 @@ private ColumnChunkPageReadStore internalReadRowGroup(int blockIndex) throws IOE
     }
     // actually read all the chunks
     ChunkListBuilder builder = new ChunkListBuilder(block.getRowCount());
-    readAllPartsVectoredOrNormal(allParts, builder);
-    rowGroup.setReleaser(builder.releaser);
-    for (Chunk chunk : builder.build()) {
-      readChunkPages(chunk, block, rowGroup);
+    try {
+      readAllPartsVectoredOrNormal(allParts, builder);
+      rowGroup.setReleaser(builder.releaser);
+      for (Chunk chunk : builder.build()) {
+        readChunkPages(chunk, block, rowGroup);
+      }
+    } catch (RuntimeException | IOException e) {
+      // If we fail before the releaser is transferred to the row group (e.g. a vectored range
+      // times out after earlier ranges already registered their buffers), release any buffers
+      // that were registered so far so that partially-read row groups do not leak.
+      builder.releaser.close();
+      throw e;
     }
 
     return rowGroup;
@@ -1464,10 +1472,18 @@ private ColumnChunkPageReadStore internalReadFilteredRowGroup(
         }
       }
     }
-    readAllPartsVectoredOrNormal(allParts, builder);
-    rowGroup.setReleaser(builder.releaser);
-    for (Chunk chunk : builder.build()) {
-      readChunkPages(chunk, block, rowGroup);
+    try {
+      readAllPartsVectoredOrNormal(allParts, builder);
+      rowGroup.setReleaser(builder.releaser);
+      for (Chunk chunk : builder.build()) {
+        readChunkPages(chunk, block, rowGroup);
+      }
+    } catch (RuntimeException | IOException e) {
+      // If we fail before the releaser is transferred to the row group (e.g. a vectored range
+      // times out after earlier ranges already registered their buffers), release any buffers
+      // that were registered so far so that partially-read row groups do not leak.
+      builder.releaser.close();
+      throw e;
     }
 
     return rowGroup;
@@ -2368,6 +2384,10 @@ public void readFromVectoredRange(ParquetFileRange currRange, ChunkListBuilder b
         LOG.error(error, e);
         throw new IOException(error, e);
       }
+      // Release the vectored-read buffer back to the allocator when the row group is closed.
+      // Requires fs.file.checksum.verify=false so the returned buffer is the allocator buffer
+      // rather than a sliced subset (see Hadoop's fs.file.checksum.verify docs).
+      builder.addBuffersToRelease(Collections.singletonList(buffer));
       ByteBufferInputStream stream = ByteBufferInputStream.wrap(buffer);
       for (ChunkDescriptor descriptor : chunks) {
         builder.add(descriptor, stream.sliceBuffers(descriptor.size), f);

parquet-hadoop/src/test/resources/core-site.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration>
+  <property>
+    <name>fs.file.checksum.verify</name>
+    <value>false</value>
+    <description>
+      Disable checksum verification on the local file system used by tests.
+      Hadoop's ChecksumFileSystem.readVectored allocates checksum buffers via
+      the caller-supplied ByteBufferAllocator without releasing them; the
+      leaked buffers trip TrackingByteBufferAllocator leak detection in tests.
+      Turning off checksum verification skips the checksum read path entirely
+      and avoids the leak.
+    </description>
+  </property>
+</configuration>

pom.xml

@@ -83,7 +83,7 @@
     <spotless.version>2.46.1</spotless.version>
     <shade.prefix>shaded.parquet</shade.prefix>
     <!-- Guarantees no newer classes/methods/constants are used by parquet. -->
-    <hadoop.version>3.3.0</hadoop.version>
+    <hadoop.version>3.4.2</hadoop.version>
     <parquet.format.version>2.13.0</parquet.format.version>
     <previous.version>1.17.0</previous.version>
     <thrift.executable>thrift</thrift.executable>