Skip to content

[GR-76116] Migrate ArrowSupport to NativeAccess. #943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
graalvmbot wants to merge 3 commits into from
Closed

[GR-76116] Migrate ArrowSupport to NativeAccess. #943

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fa4ee4d
Allow void C API upcall targets
fangerer Jun 1, 2026
ad88eee
Migrate Arrow callbacks to NativeAccess
fangerer Jun 1, 2026
84660d0
Remove unused NativeLibrary and FunctionWithSignature
fangerer Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...e.graal.python.processor/src/com/oracle/graal/python/processor/CApiBuiltinsProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,8 @@ private static String toJNIName(TypeMirror typeMirror) {
}

private boolean verifySignatureOfExplicitUpcallTarget(ExecutableElement explicitUpcallTarget) {
if (!explicitUpcallTarget.getReturnType().getKind().isPrimitive()) {
TypeKind returnKind = explicitUpcallTarget.getReturnType().getKind();
if (returnKind != TypeKind.VOID && !returnKind.isPrimitive()) {
processingEnv.getMessager().printError("Return type must be primitive but was " + explicitUpcallTarget.getReturnType(), explicitUpcallTarget);
return false;
}
Expand Down
4 changes: 2 additions & 2 deletions ....graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1537,11 +1537,11 @@ static PTuple doCreate(long arrowArrayAddr, long arrowSchemaAddr,
@Bind Node inliningTarget,
@Cached PythonCextCapsuleBuiltins.PyCapsuleNewNode pyCapsuleNewNode) {
PythonContext ctx = getContext(inliningTarget);
long arrayDestructor = ctx.arrowSupport.getArrowArrayDestructor(inliningTarget);
long arrayDestructor = ctx.arrowSupport.getArrowArrayDestructor();
long arrayCapsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowArray.CAPSULE_NAME, true);
PyCapsule arrowArrayCapsule = pyCapsuleNewNode.execute(inliningTarget, arrowArrayAddr, arrayCapsuleNamePointer, arrayDestructor);

long schemaDestructor = ctx.arrowSupport.getArrowSchemaDestructor(inliningTarget);
long schemaDestructor = ctx.arrowSupport.getArrowSchemaDestructor();
long schemaCapsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowSchema.CAPSULE_NAME, true);
PyCapsule arrowSchemaCapsule = pyCapsuleNewNode.execute(inliningTarget, arrowSchemaAddr, schemaCapsuleNamePointer, schemaDestructor);
return PFactory.createTuple(ctx.getLanguage(inliningTarget), new Object[]{arrowSchemaCapsule, arrowArrayCapsule});
Expand Down
5 changes: 1 addition & 4 deletions ....graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/NativePointer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,7 @@
import com.oracle.truffle.api.library.ExportLibrary;
import com.oracle.truffle.api.library.ExportMessage;

/**
* Currently only used for wrapping pointers to call TruffleString, see GR-71311 and in
* InvokeArrowReleaseCallbackNode which still uses original NFI.
*/
/** Currently only used for wrapping pointers to call TruffleString, see GR-71311. */
@ExportLibrary(InteropLibrary.class)
public final class NativePointer implements TruffleObject {
private final long ptr;
Expand Down
29 changes: 18 additions & 11 deletions ...graal/python/runtime/arrow/ArrowUtil.java → ...hon/nodes/arrow/ArrowReleaseCallback.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* The Universal Permissive License (UPL), Version 1.0
Expand Down Expand Up @@ -38,19 +38,26 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package com.oracle.graal.python.runtime.arrow;
package com.oracle.graal.python.nodes.arrow;

import static com.oracle.graal.python.nodes.StringLiterals.J_NFI_LANGUAGE;
import static com.oracle.graal.python.util.PythonUtils.callCallTarget;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodType;

import com.oracle.graal.python.runtime.PythonContext;
import com.oracle.truffle.api.nodes.Node;
import com.oracle.truffle.api.source.Source;
import com.oracle.graal.python.runtime.nativeaccess.NativeAccessSupport;
import com.oracle.truffle.api.CompilerDirectives;

public class ArrowUtil {
public final class ArrowReleaseCallback {
private static final MethodHandle HANDLE = NativeAccessSupport.createDowncallHandle(
MethodType.methodType(void.class, long.class, long.class), false);

public static Object createNfiSignature(Node location, String methodSignature, PythonContext ctx) {
Source sigSource = Source.newBuilder(J_NFI_LANGUAGE, methodSignature, "python-nfi-signature").build();
return callCallTarget(ctx.getEnv().parseInternal(sigSource), location);
private ArrowReleaseCallback() {
}

public static void execute(long releaseCallback, long baseStructure) {
try {
HANDLE.invokeExact(releaseCallback, baseStructure);
} catch (Throwable e) {
throw CompilerDirectives.shouldNotReachHere("Unable to call release callback. Error:", e);
}
}
}
102 changes: 0 additions & 102 deletions ....graal.python/src/com/oracle/graal/python/nodes/arrow/InvokeArrowReleaseCallbackNode.java
View file
Open in desktop

This file was deleted.

90 changes: 46 additions & 44 deletions ...l.python/src/com/oracle/graal/python/nodes/arrow/capsule/ArrowArrayCapsuleDestructor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,60 +40,62 @@
*/
package com.oracle.graal.python.nodes.arrow.capsule;

import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;

import com.oracle.graal.python.annotations.CApiUpcallTarget;
import com.oracle.graal.python.builtins.modules.cext.PythonCextCapsuleBuiltins.PyCapsuleGetPointerNode;
import com.oracle.graal.python.builtins.objects.PNone;
import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.NativeToPythonNode;
import com.oracle.graal.python.runtime.nativeaccess.NativeMemory;
import com.oracle.graal.python.nodes.arrow.ArrowArray;
import com.oracle.graal.python.nodes.arrow.InvokeArrowReleaseCallbackNode;
import com.oracle.graal.python.nodes.arrow.ArrowReleaseCallback;
import com.oracle.graal.python.runtime.PythonContext;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.dsl.Bind;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.interop.InteropLibrary;
import com.oracle.truffle.api.interop.TruffleObject;
import com.oracle.truffle.api.library.CachedLibrary;
import com.oracle.truffle.api.library.ExportLibrary;
import com.oracle.truffle.api.library.ExportMessage;
import com.oracle.truffle.api.nodes.Node;
import com.oracle.graal.python.runtime.nativeaccess.NativeMemory;
import com.oracle.truffle.api.CompilerAsserts;

public final class ArrowArrayCapsuleDestructor {
private static final MethodHandle HANDLE_EXECUTE;

@ExportLibrary(InteropLibrary.class)
public class ArrowArrayCapsuleDestructor implements TruffleObject {
static {
try {
HANDLE_EXECUTE = MethodHandles.lookup().findStatic(ArrowArrayCapsuleDestructor.class, "execute",
MethodType.methodType(void.class, long.class));
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}

@ExportMessage
boolean isExecutable() {
return true;
private ArrowArrayCapsuleDestructor() {
}

@ExportMessage
Object execute(Object[] args,
@Bind Node inliningTarget,
@CachedLibrary(limit = "1") InteropLibrary lib,
@Cached NativeToPythonNode nativeToPythonNode,
@Cached PyCapsuleGetPointerNode capsuleGetPointerNode,
@Cached InvokeArrowReleaseCallbackNode.Lazy invokeReleaseCallbackNode) {
if (args.length != 1 || !lib.isPointer(args[0])) {
throw CompilerDirectives.shouldNotReachHere();
}
public static MethodHandle getMethodHandle() {
return HANDLE_EXECUTE;
}

Object capsule = nativeToPythonNode.execute(args[0]);
PythonContext ctx = PythonContext.get(inliningTarget);
@CApiUpcallTarget
private static void execute(long capsulePointer) {
CompilerAsserts.neverPartOfCompilation();
PythonContext ctx = PythonContext.get(null);
ctx.ensureNativeAccess();
long capsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowArray.CAPSULE_NAME, true);
var arrowArray = ArrowArray.wrap(capsuleGetPointerNode.execute(inliningTarget, capsule, capsuleNamePointer));
/*
* The exported PyCapsules should have a destructor that calls the release callback of the
* Arrow struct, if it is not already null. This prevents a memory leak in case the capsule
* was never passed to another consumer.
*
* For more information see:
* https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#lifetime-
* semantics
*/
if (!arrowArray.isReleased()) {
invokeReleaseCallbackNode.get(inliningTarget).executeCached(arrowArray.releaseCallback(), arrowArray.memoryAddress());
Object capsule = NativeToPythonNode.executeRawUncached(capsulePointer);
long capsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowArray.CAPSULE_NAME, false);
try {
var arrowArray = ArrowArray.wrap(PyCapsuleGetPointerNode.executeUncached(capsule, capsuleNamePointer));
/*
* The exported PyCapsules should have a destructor that calls the release callback of
* the Arrow struct, if it is not already null. This prevents a memory leak in case the
* capsule was never passed to another consumer.
*
* For more information see:
* https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#lifetime-
* semantics
*/
if (!arrowArray.isReleased()) {
ArrowReleaseCallback.execute(arrowArray.releaseCallback(), arrowArray.memoryAddress());
}
NativeMemory.free(arrowArray.memoryAddress());
} finally {
NativeMemory.free(capsuleNamePointer);
}
NativeMemory.free(arrowArray.memoryAddress());
return PNone.NO_VALUE;
}
}
74 changes: 38 additions & 36 deletions ....python/src/com/oracle/graal/python/nodes/arrow/capsule/ArrowSchemaCapsuleDestructor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,53 +40,55 @@
*/
package com.oracle.graal.python.nodes.arrow.capsule;

import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;

import com.oracle.graal.python.annotations.CApiUpcallTarget;
import com.oracle.graal.python.builtins.modules.cext.PythonCextCapsuleBuiltins.PyCapsuleGetPointerNode;
import com.oracle.graal.python.builtins.objects.PNone;
import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.NativeToPythonNode;
import com.oracle.graal.python.runtime.nativeaccess.NativeMemory;
import com.oracle.graal.python.nodes.arrow.ArrowReleaseCallback;
import com.oracle.graal.python.nodes.arrow.ArrowSchema;
import com.oracle.graal.python.nodes.arrow.InvokeArrowReleaseCallbackNode;
import com.oracle.graal.python.runtime.PythonContext;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.dsl.Bind;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.interop.InteropLibrary;
import com.oracle.truffle.api.interop.TruffleObject;
import com.oracle.truffle.api.library.CachedLibrary;
import com.oracle.truffle.api.library.ExportLibrary;
import com.oracle.truffle.api.library.ExportMessage;
import com.oracle.truffle.api.nodes.Node;
import com.oracle.graal.python.runtime.nativeaccess.NativeMemory;
import com.oracle.truffle.api.CompilerAsserts;

@ExportLibrary(InteropLibrary.class)
public class ArrowSchemaCapsuleDestructor implements TruffleObject {
public final class ArrowSchemaCapsuleDestructor {
private static final MethodHandle HANDLE_EXECUTE;

@ExportMessage
boolean isExecutable() {
return true;
static {
try {
HANDLE_EXECUTE = MethodHandles.lookup().findStatic(ArrowSchemaCapsuleDestructor.class, "execute",
MethodType.methodType(void.class, long.class));
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}

@ExportMessage
Object execute(Object[] args,
@Bind Node inliningTarget,
@CachedLibrary(limit = "1") InteropLibrary lib,
@Cached NativeToPythonNode nativeToPythonNode,
@Cached PyCapsuleGetPointerNode pyCapsuleGetPointerNode,
@Cached InvokeArrowReleaseCallbackNode.Lazy invokeReleaseCallbackNode) {
if (args.length != 1 || !lib.isPointer(args[0])) {
throw CompilerDirectives.shouldNotReachHere();
}
private ArrowSchemaCapsuleDestructor() {
}

public static MethodHandle getMethodHandle() {
return HANDLE_EXECUTE;
}

Object capsule = nativeToPythonNode.execute(args[0]);
PythonContext ctx = PythonContext.get(inliningTarget);
@CApiUpcallTarget
private static void execute(long capsulePointer) {
CompilerAsserts.neverPartOfCompilation();
PythonContext ctx = PythonContext.get(null);
ctx.ensureNativeAccess();
long capsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowSchema.CAPSULE_NAME, true);
var arrowSchema = ArrowSchema.wrap(pyCapsuleGetPointerNode.execute(inliningTarget, capsule, capsuleNamePointer));
Object capsule = NativeToPythonNode.executeRawUncached(capsulePointer);
long capsuleNamePointer = ctx.stringToNativeUtf8Bytes(ArrowSchema.CAPSULE_NAME, false);
try {
var arrowSchema = ArrowSchema.wrap(PyCapsuleGetPointerNode.executeUncached(capsule, capsuleNamePointer));

if (!arrowSchema.isReleased()) {
invokeReleaseCallbackNode.get(inliningTarget).executeCached(arrowSchema.releaseCallback(), arrowSchema.memoryAddress());
}
if (!arrowSchema.isReleased()) {
ArrowReleaseCallback.execute(arrowSchema.releaseCallback(), arrowSchema.memoryAddress());
}

NativeMemory.free(arrowSchema.memoryAddress());
return PNone.NO_VALUE;
NativeMemory.free(arrowSchema.memoryAddress());
} finally {
NativeMemory.free(capsuleNamePointer);
}
}
}
Loading
Loading