diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6b69758..8598d5f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,8 @@ jobs: - "fedora-44-amd64" - "gentoo" - "ubuntu-22.04-jammy-amd64" - - "ubuntu-22.04-jammy-amd64-valgrind" - "ubuntu-24.04-noble-amd64" + - "ubuntu-24.04-noble-amd64-valgrind" - "ubuntu-26.04-resolute-amd64" # has a dependency on the test image - "manylinux_2_28-wheel-build" diff --git a/Makefile b/Makefile index b0fe9468..35cb8973 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ TARGETS = \ ubuntu-22.04-jammy-amd64 \ ubuntu-24.04-noble-amd64 \ ubuntu-26.04-resolute-amd64 \ - ubuntu-22.04-jammy-amd64-valgrind \ + ubuntu-24.04-noble-amd64-valgrind \ ubuntu-26.04-resolute-arm64v8 \ ubuntu-26.04-resolute-ppc64le \ ubuntu-26.04-resolute-s390x diff --git a/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile b/archive/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/Dockerfile rename to archive/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile diff --git a/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile-dbg b/archive/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile-dbg similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/Dockerfile-dbg rename to archive/ubuntu-22.04-jammy-amd64-valgrind/Dockerfile-dbg diff --git a/ubuntu-22.04-jammy-amd64-valgrind/Makefile b/archive/ubuntu-22.04-jammy-amd64-valgrind/Makefile similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/Makefile rename to archive/ubuntu-22.04-jammy-amd64-valgrind/Makefile diff --git a/ubuntu-22.04-jammy-amd64-valgrind/README-dbg.md b/archive/ubuntu-22.04-jammy-amd64-valgrind/README-dbg.md similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/README-dbg.md rename to archive/ubuntu-22.04-jammy-amd64-valgrind/README-dbg.md diff --git a/ubuntu-22.04-jammy-amd64-valgrind/python.supp b/archive/ubuntu-22.04-jammy-amd64-valgrind/python.supp similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/python.supp rename to archive/ubuntu-22.04-jammy-amd64-valgrind/python.supp diff --git a/ubuntu-22.04-jammy-amd64-valgrind/setup.sh b/archive/ubuntu-22.04-jammy-amd64-valgrind/setup.sh similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/setup.sh rename to archive/ubuntu-22.04-jammy-amd64-valgrind/setup.sh diff --git a/ubuntu-22.04-jammy-amd64-valgrind/test.sh b/archive/ubuntu-22.04-jammy-amd64-valgrind/test.sh similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/test.sh rename to archive/ubuntu-22.04-jammy-amd64-valgrind/test.sh diff --git a/ubuntu-22.04-jammy-amd64-valgrind/update.sh b/archive/ubuntu-22.04-jammy-amd64-valgrind/update.sh similarity index 100% rename from ubuntu-22.04-jammy-amd64-valgrind/update.sh rename to archive/ubuntu-22.04-jammy-amd64-valgrind/update.sh diff --git a/ubuntu-24.04-noble-amd64-valgrind/Dockerfile b/ubuntu-24.04-noble-amd64-valgrind/Dockerfile new file mode 100644 index 00000000..39d8ccef --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/Dockerfile @@ -0,0 +1,61 @@ +FROM ubuntu:noble + +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \ + cmake \ + g++ \ + ghostscript \ + git \ + libfreetype6-dev \ + libfribidi-dev \ + libharfbuzz-dev \ + libimagequant-dev \ + libjpeg-turbo8-dev \ + liblcms2-dev \ + libopenjp2-7-dev \ + libssl-dev \ + libtiff5-dev \ + libwebp-dev \ + meson \ + nasm \ + netpbm \ + python3.12-dbg \ + python3-dev \ + python3-numpy \ + python3-setuptools \ + python3-tk \ + sudo \ + tcl8.6-dev \ + tk8.6-dev \ + virtualenv \ + wget \ + xvfb \ + zlib1g-dev \ + valgrind \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN useradd --uid 1001 pillow \ + && mkdir /home/pillow \ + && chown pillow:pillow /home/pillow + +ARG PIP_DISABLE_PIP_VERSION_CHECK=1 +ARG PIP_NO_CACHE_DIR=1 + +RUN virtualenv -p /usr/bin/python3.12-dbg --system-site-packages /vpy3 \ + && /vpy3/bin/pip install --upgrade pip \ + && /vpy3/bin/pip install olefile pytest pytest-cov pytest-timeout pytest-valgrind \ + && chown -R pillow:pillow /vpy3 + +COPY depends /depends +RUN cd /depends \ + && ./install_libavif.sh \ + && ./install_raqm.sh + +COPY python.supp /depends + +ENV FORCE_COLOR=1 + +USER pillow +CMD ["depends/test.sh"] + +#docker run -v $TRAVIS_BUILD_DIR:/Pillow pythonpillow/ubuntu-24.04-noble-amd64 diff --git a/ubuntu-24.04-noble-amd64-valgrind/Dockerfile-dbg b/ubuntu-24.04-noble-amd64-valgrind/Dockerfile-dbg new file mode 100644 index 00000000..b539ac04 --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/Dockerfile-dbg @@ -0,0 +1,27 @@ +FROM ubuntu-24.04-noble-amd64-valgrind + +USER root +RUN apt-get -qq update && apt-get install -y ubuntu-dbgsym-keyring + +RUN echo "deb http://ddebs.ubuntu.com noble main restricted universe multiverse" > /etc/apt/sources.list.d/ddebs.list +RUN echo "deb http://ddebs.ubuntu.com noble-updates main restricted universe multiverse" >> /etc/apt/sources.list.d/ddebs.list +RUN perl -pi -e 's/# deb-src/deb-src/;' /etc/apt/sources.list + +RUN apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \ + libfreetype6-dbgsym \ + libtiff5-dbgsym \ + libwebpmux3-dbgsym \ + libwebp7-dbgsym \ + libwebpdemux2-dbgsym \ + libwebpmux3-dbgsym \ + libopenjp2-7-dbgsym \ + libharfbuzz0b-dbgsym \ + libfribidi0-dbgsym \ + emacs-nox + +RUN cd /usr/src && apt-get source freetype tiff libwebp libopenjp2-7 harfbuzz fribidi + +COPY setup.sh /depends/setup.sh +USER pillow +CMD ["/depends/setup.sh"] diff --git a/ubuntu-24.04-noble-amd64-valgrind/Makefile b/ubuntu-24.04-noble-amd64-valgrind/Makefile new file mode 100644 index 00000000..096cebcd --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/Makefile @@ -0,0 +1,38 @@ +WD = $(shell pwd) +USERNAME := $(shell docker info | grep Username | xargs | cut -d ' ' -f 2) +TARGET := $(notdir $(WD)) +ROOT := $(abspath $(WD)/../Pillow) +IMAGENAME := $(if $(USERNAME), $(USERNAME)/$(TARGET), $(TARGET)) +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) + +.PHONY: build +build: + cp -r ../Pillow/depends . + cp test.sh depends + docker build -t $(IMAGENAME):$(BRANCH) . + +.PHONY: update +update: + ./update.sh + +.PHONY: test +test: + docker run --rm -e "PILLOW_VALGRIND_TEST=true" -v $(ROOT):/Pillow $(IMAGENAME):$(BRANCH) + +.PHONY: push +push: + docker push $(IMAGENAME):$(BRANCH) + +.PHONY: clean +clean: + rm -r depends + +.PHONY: shell +shell: + docker run --rm -it -v $(ROOT):/Pillow $(IMAGENAME):$(BRANCH) /bin/bash + +build-dbg: + docker build -t $(IMAGENAME)-dbg:$(BRANCH) -f Dockerfile-dbg . + +dbg: + docker run --privileged --rm -it -v $(ROOT):/Pillow $(IMAGENAME)-dbg:$(BRANCH) diff --git a/ubuntu-24.04-noble-amd64-valgrind/README-dbg.md b/ubuntu-24.04-noble-amd64-valgrind/README-dbg.md new file mode 100644 index 00000000..f983c705 --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/README-dbg.md @@ -0,0 +1,52 @@ +# Debug image + +Sometimes we'll get an error that's most easily reproduced in the +valgrind docker image. [citation needed] + +The `build-dbg` and `dbg` shell provide some symbols and debugging +tools to fix that. Emacs is provided, as are some of the source packages. + +You may need to set source directories with the `set substitute-path` +command. + +e.g.: + +``` +pillow@2f7a4ea5c959:/Pillow/oss-fuzz-tests$ gdb python +GNU gdb (Ubuntu 12.1-0ubuntu1~22.04) 12.1 +Copyright (C) 2022 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +Type "show copying" and "show warranty" for details. +This GDB was configured as "x86_64-linux-gnu". +Type "show configuration" for configuration details. +For bug reporting instructions, please see: +. +Find the GDB manual and other documentation resources online at: + . + +For help, type "help". +Type "apropos word" to search for commands related to "word"... +Reading symbols from python... +(gdb) b _imagingft.c:922 +No source file named _imagingft.c. +Make breakpoint pending on future shared library load? (y or [n]) y +Breakpoint 1 (_imagingft.c:922) pending. +(gdb) r f.py DejaVuSans-24-8-stripped.ttf +Starting program: /vpy3/bin/python f.py DejaVuSans-24-8-stripped.ttf +[Thread debugging using libthread_db enabled] +Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". + +Breakpoint 1, font_render (self=0x7ffff70e9780, args=) at src/_imagingft.c:922 +922 if (!bitmap.buffer) { +... +0x00007ffff6fe4d6a in FT_Bitmap_Convert (library=, source=source@entry=0x7fffffffdff0, target=target@entry=0x7fffffffe020, alignment=alignment@entry=1) at ./src/base/ftbitmap.c:633 +633 ./src/base/ftbitmap.c: No such file or directory. +(gdb) set substitute-path ./src /usr/src/freetype-2.11.1+dfsg/src/ +(gdb) f +#0 0x00007ffff6fe4d6a in FT_Bitmap_Convert (library=, source=source@entry=0x7fffffffdff0, target=target@entry=0x7fffffffe020, + alignment=alignment@entry=1) at ./src/base/ftbitmap.c:633 +633 FT_Int val = *ss; + +``` diff --git a/ubuntu-24.04-noble-amd64-valgrind/python.supp b/ubuntu-24.04-noble-amd64-valgrind/python.supp new file mode 100644 index 00000000..21be222e --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/python.supp @@ -0,0 +1,6 @@ +{ + + Memcheck:Cond + ... + fun:encode_current_locale +} diff --git a/ubuntu-24.04-noble-amd64-valgrind/setup.sh b/ubuntu-24.04-noble-amd64-valgrind/setup.sh new file mode 100755 index 00000000..83cbaa8f --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +source /vpy3/bin/activate +cd /Pillow +make clean +make install + +bash diff --git a/ubuntu-24.04-noble-amd64-valgrind/test.sh b/ubuntu-24.04-noble-amd64-valgrind/test.sh new file mode 100755 index 00000000..b0a7aa38 --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/test.sh @@ -0,0 +1,16 @@ +#!/bin/bash +source /vpy3/bin/activate +cd /Pillow +make clean +make install-coverage + +PYTHONMALLOC=malloc valgrind --suppressions=/depends/python.supp --leak-check=no \ + --log-file=/tmp/valgrind-output \ + python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output + +# To run one test in the image: +# make bash +# PYTHONMALLOC=malloc valgrind --leak-check=no --suppressions=/depends/python.supp pytest -vv --valgrind --no-memcheck Tests/test_file_eps.py + +# to run one test with GDB support: +# PYTHONMALLOC=malloc valgrind --vgdb-error=0 --suppressions=../../test/python.supp pytest Tests/test_file_libtiff.py::TestFileLibTiff::test_custom_metadata diff --git a/ubuntu-24.04-noble-amd64-valgrind/update.sh b/ubuntu-24.04-noble-amd64-valgrind/update.sh new file mode 100755 index 00000000..b5962bec --- /dev/null +++ b/ubuntu-24.04-noble-amd64-valgrind/update.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker pull ubuntu:noble