Browse Source

Version v0.3.0-alpha.13-shell-out-r4

asonix/shell-out v0.3.0-alpha.13-shell-out-r4
Aode (lion) 2 months ago
parent
commit
db95c85bdf
  1. 111
      docker/dev/Dockerfile.amd64
  2. 112
      docker/dev/Dockerfile.arm32v7
  3. 112
      docker/dev/Dockerfile.arm64v8
  4. 2
      docker/prod/docker-compose.yml
  5. 74
      src/exiv2.rs
  6. 71
      src/magick.rs
  7. 2
      src/upload_manager.rs

111
docker/dev/Dockerfile.amd64

@ -1,5 +1,5 @@
# Basic cross-build environment
FROM ubuntu:20.04 as cross-build
FROM rustembedded/cross:x86_64-unknown-linux-musl as cross-build
ARG UID=1000
ARG GID=1000
@ -7,23 +7,10 @@ ARG GID=1000
ENV \
ARCH=amd64 \
HOST=x86_64-unknown-linux \
TOOL=x86_64-linux-gnu \
TARGET=x86_64-unknown-linux-gnu \
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
CC_X86_64_UNKNOWN_LINUX_GNU=x86_64-linux-gnu-gcc \
CXX_X86_64_UNKNOWN_LINUX_GNU=x86_64-linux-gnu-g++ \
BUILD_MODE=release
ENV \
TOOLCHAIN=stable \
DEBIAN_FRONTEND=noninteractive \
PKG_CONFIG_ALLOW_CROSS=1 \
PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/pkgconfig \
LD_LIBRARY_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LD_RUN_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LDFLAGS="-L/usr/lib/$TOOL -L/usr/$TOOL/lib -Wl,-rpath-link,/usr/lib/$TOOL -Wl,-rpath-link,/usr/$TOOL/lib" \
CFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include" \
CPPFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include"
TARGET=x86_64-unknown-linux-musl \
BUILD_MODE=release \
DEBIAN_FRONTEND=noninteractive
RUN \
sed 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=amd64,i386] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g' /etc/apt/sources.list > /etc/apt/sources.list.d/ports.list && \
@ -35,72 +22,14 @@ RUN \
--ingroup build \
--uid $UID \
--home /opt/build \
build && \
dpkg --add-architecture $ARCH && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y \
pkg-config \
build-essential \
crossbuild-essential-$ARCH
build
WORKDIR /opt/build
# Environment for ImageMagick
FROM cross-build as imagemagick-builder
RUN \
apt-get install -y \
libltdl-dev:$ARCH \
libjpeg-dev:$ARCH \
libpng-dev:$ARCH \
libwebp-dev:$ARCH \
liblzma-dev:$ARCH \
libxml2-dev:$ARCH
ADD --chown=build:build https://imagemagick.org/download/ImageMagick.tar.gz /opt/build/ImageMagick.tar.gz
USER build
RUN \
tar zxf ImageMagick.tar.gz && \
mv ImageMagick-* ImageMagick
WORKDIR /opt/build/ImageMagick
RUN \
./configure \
CC=$TOOL-gcc \
CXX=$TOOL-g++ \
--enable-shared \
--with-modules \
--disable-static \
--disable-docs \
--prefix=/usr/local \
--with-utilities=yes \
--with-magick-plus-plus=no \
--without-perl \
--with-xml=yes \
--with-png=yes \
--with-jpeg=yes \
--with-webp=yes \
--host=$HOST && \
make
USER root
RUN \
make install && \
ldconfig /usr/local/lib
# Environment for Rust
FROM cross-build as rust
RUN \
apt-get install -y curl
ENV \
PATH=$PATH:/opt/build/.cargo/bin
@ -113,43 +42,15 @@ RUN \
./rustup.sh --default-toolchain $TOOLCHAIN --profile minimal -y && \
rustup target add $TARGET
USER root
# Environment for pict-rs
FROM cross-build as pict-rs-builder
RUN \
apt-get install -y \
exiv2:$ARCH \
libxml2:$ARCH \
libltdl7:$ARCH \
llvm-dev \
libclang-dev \
clang \
libpng16-16:$ARCH \
libjpeg8:$ARCH \
libwebp6:$ARCH \
libwebpdemux2:$ARCH \
libwebpmux3:$ARCH \
libgomp1:$ARCH \
ffmpeg:$ARCH
ENV \
PATH=$PATH:/opt/build/.cargo/bin \
PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig \
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib \
LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib \
LDFLAGS="$LDFLAGS -L/usr/local/lib" \
IMAGE_MAGICK_LIB_DIRS=/usr/local/lib \
IMAGE_MAGICK_INCLUDE_DIRS=/usr/local/include/ImageMagick-7 \
CFLAGS="$CFLAGS -I/usr/local/include/ImageMagick-7" \
CPPFLAGS="$CPPFLAGS -I/usr/local/include/ImageMagick-7" \
RUSTFLAGS="-L/usr/lib/$TOOL -C link-arg=-Wl,-rpath-link,/usr/lib/$TOOL -L/usr/$TOOL/lib -C link-arg=-Wl,-rpath-link,/usr/$TOOL/lib"
PATH=$PATH:/opt/build/.cargo/bin
COPY --from=rust --chown=build:build /opt/build/.cargo /opt/build/.cargo
COPY --from=rust --chown=build:build /opt/build/.rustup /opt/build/.rustup
COPY --from=imagemagick-builder /usr/local/ /usr/local
COPY root/ /
USER build

112
docker/dev/Dockerfile.arm32v7

@ -1,5 +1,5 @@
# Basic cross-build environment
FROM ubuntu:20.04 as cross-build
FROM rustembedded/cross:armv7-unknown-linux-musleabihf as cross-build
ARG UID=1000
ARG GID=1000
@ -7,23 +7,10 @@ ARG GID=1000
ENV \
ARCH=armhf \
HOST=arm-unknown-linux \
TOOL=arm-linux-gnueabihf \
TARGET=armv7-unknown-linux-gnueabihf \
CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \
CC_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc \
CXX_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-g++ \
BUILD_MODE=release
ENV \
TOOLCHAIN=stable \
DEBIAN_FRONTEND=noninteractive \
PKG_CONFIG_ALLOW_CROSS=1 \
PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/pkgconfig \
LD_LIBRARY_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LD_RUN_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LDFLAGS="-L/usr/lib/$TOOL -L/usr/$TOOL/lib -Wl,-rpath-link,/usr/lib/$TOOL -Wl,-rpath-link,/usr/$TOOL/lib" \
CFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include" \
CPPFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include"
TARGET=armv7-unknown-linux-musleabihf \
BUILD_MODE=release \
DEBIAN_FRONTEND=noninteractive
RUN \
sed 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=amd64,i386] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g' /etc/apt/sources.list > /etc/apt/sources.list.d/ports.list && \
@ -35,72 +22,14 @@ RUN \
--ingroup build \
--uid $UID \
--home /opt/build \
build && \
dpkg --add-architecture $ARCH && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y \
pkg-config \
build-essential \
crossbuild-essential-$ARCH
build
WORKDIR /opt/build
# Environment for ImageMagick
FROM cross-build as imagemagick-builder
RUN \
apt-get install -y \
libltdl-dev:$ARCH \
libjpeg-dev:$ARCH \
libpng-dev:$ARCH \
libwebp-dev:$ARCH \
liblzma-dev:$ARCH \
libxml2-dev:$ARCH
ADD --chown=build:build https://imagemagick.org/download/ImageMagick.tar.gz /opt/build/ImageMagick.tar.gz
USER build
RUN \
tar zxf ImageMagick.tar.gz && \
mv ImageMagick-* ImageMagick
WORKDIR /opt/build/ImageMagick
RUN \
./configure \
CC=$TOOL-gcc \
CXX=$TOOL-g++ \
--enable-shared \
--with-modules \
--disable-static \
--disable-docs \
--prefix=/usr/local \
--with-utilities=yes \
--with-magick-plus-plus=no \
--without-perl \
--with-xml=yes \
--with-png=yes \
--with-jpeg=yes \
--with-webp=yes \
--host=$HOST && \
make
USER root
RUN \
make install && \
ldconfig /usr/local/lib
# Environment for Rust
FROM cross-build as rust
RUN \
apt-get install -y curl
ENV \
PATH=$PATH:/opt/build/.cargo/bin
@ -113,44 +42,15 @@ RUN \
./rustup.sh --default-toolchain $TOOLCHAIN --profile minimal -y && \
rustup target add $TARGET
USER root
# Environment for pict-rs
FROM cross-build as pict-rs-builder
RUN \
apt-get install -y \
exiv2:$ARCH \
libxml2:$ARCH \
libltdl7:$ARCH \
llvm-dev \
libclang-dev \
clang \
libpng16-16:$ARCH \
libjpeg8:$ARCH \
libwebp6:$ARCH \
libwebpdemux2:$ARCH \
libwebpmux3:$ARCH \
libgomp1:$ARCH \
ffmpeg:$ARCH && \
rm -rf /usr/include/x86_64-linux-gnu
ENV \
PATH=$PATH:/opt/build/.cargo/bin \
PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig \
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib \
LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib \
LDFLAGS="$LDFLAGS -L/usr/local/lib" \
IMAGE_MAGICK_LIB_DIRS=/usr/local/lib \
IMAGE_MAGICK_INCLUDE_DIRS=/usr/local/include/ImageMagick-7 \
CFLAGS="$CFLAGS -I/usr/local/include/ImageMagick-7" \
CPPFLAGS="$CPPFLAGS -I/usr/local/include/ImageMagick-7" \
RUSTFLAGS="-L/usr/lib/$TOOL -C link-arg=-Wl,-rpath-link,/usr/lib/$TOOL -L/usr/$TOOL/lib -C link-arg=-Wl,-rpath-link,/usr/$TOOL/lib"
PATH=$PATH:/opt/build/.cargo/bin
COPY --from=rust --chown=build:build /opt/build/.cargo /opt/build/.cargo
COPY --from=rust --chown=build:build /opt/build/.rustup /opt/build/.rustup
COPY --from=imagemagick-builder /usr/local/ /usr/local
COPY root/ /
USER build

112
docker/dev/Dockerfile.arm64v8

@ -1,5 +1,5 @@
# Basic cross-build environment
FROM ubuntu:20.04 as cross-build
FROM rustembedded/cross:aarch64-unknown-linux-musl as cross-build
ARG UID=1000
ARG GID=1000
@ -7,23 +7,10 @@ ARG GID=1000
ENV \
ARCH=arm64 \
HOST=aarch64-unknown-linux \
TOOL=aarch64-linux-gnu \
TARGET=aarch64-unknown-linux-gnu \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CC_AARCH64_UNKNOWN_LINUX_GNU=aarch64-linux-gnu-gcc \
CXX_AARCH64_UNKNOWN_LINUX_GNU=aarch64-linux-gnu-g++ \
BUILD_MODE=release
ENV \
TOOLCHAIN=stable \
DEBIAN_FRONTEND=noninteractive \
PKG_CONFIG_ALLOW_CROSS=1 \
PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/pkgconfig \
LD_LIBRARY_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LD_RUN_PATH=/usr/lib/$TOOL:/usr/$TOOL/lib \
LDFLAGS="-L/usr/lib/$TOOL -L/usr/$TOOL/lib -Wl,-rpath-link,/usr/lib/$TOOL -Wl,-rpath-link,/usr/$TOOL/lib" \
CFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include" \
CPPFLAGS="-I/usr/include/$TOOL -I/usr/$TOOL/include -I/usr/include"
TARGET=aarch64-unknown-linux-musl \
BUILD_MODE=release \
DEBIAN_FRONTEND=noninteractive
RUN \
sed 's/http:\/\/\(.*\).ubuntu.com\/ubuntu\//[arch-=amd64,i386] http:\/\/ports.ubuntu.com\/ubuntu-ports\//g' /etc/apt/sources.list > /etc/apt/sources.list.d/ports.list && \
@ -35,72 +22,14 @@ RUN \
--ingroup build \
--uid $UID \
--home /opt/build \
build && \
dpkg --add-architecture $ARCH && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y \
pkg-config \
build-essential \
crossbuild-essential-$ARCH
build
WORKDIR /opt/build
# Environment for ImageMagick
FROM cross-build as imagemagick-builder
RUN \
apt-get install -y \
libltdl-dev:$ARCH \
libjpeg-dev:$ARCH \
libpng-dev:$ARCH \
libwebp-dev:$ARCH \
liblzma-dev:$ARCH \
libxml2-dev:$ARCH
ADD --chown=build:build https://imagemagick.org/download/ImageMagick.tar.gz /opt/build/ImageMagick.tar.gz
USER build
RUN \
tar zxf ImageMagick.tar.gz && \
mv ImageMagick-* ImageMagick
WORKDIR /opt/build/ImageMagick
RUN \
./configure \
CC=$TOOL-gcc \
CXX=$TOOL-g++ \
--enable-shared \
--with-modules \
--disable-static \
--disable-docs \
--prefix=/usr/local \
--with-utilities=yes \
--with-magick-plus-plus=no \
--without-perl \
--with-xml=yes \
--with-png=yes \
--with-jpeg=yes \
--with-webp=yes \
--host=$HOST && \
make
USER root
RUN \
make install && \
ldconfig /usr/local/lib
# Environment for Rust
FROM cross-build as rust
RUN \
apt-get install -y curl
ENV \
PATH=$PATH:/opt/build/.cargo/bin
@ -113,44 +42,15 @@ RUN \
./rustup.sh --default-toolchain $TOOLCHAIN --profile minimal -y && \
rustup target add $TARGET
USER root
# Environment for pict-rs
FROM cross-build as pict-rs-builder
RUN \
apt-get install -y \
exiv2:$ARCH \
libxml2:$ARCH \
libltdl7:$ARCH \
llvm-dev \
libclang-dev \
clang \
libpng16-16:$ARCH \
libjpeg8:$ARCH \
libwebp6:$ARCH \
libwebpdemux2:$ARCH \
libwebpmux3:$ARCH \
libgomp1:$ARCH \
ffmpeg && \
rm -rf /usr/include/x86_64-linux-gnu
ENV \
PATH=$PATH:/opt/build/.cargo/bin \
PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig \
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib \
LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib \
LDFLAGS="$LDFLAGS -L/usr/local/lib" \
IMAGE_MAGICK_LIB_DIRS=/usr/local/lib \
IMAGE_MAGICK_INCLUDE_DIRS=/usr/local/include/ImageMagick-7 \
CFLAGS="$CFLAGS -I/usr/local/include/ImageMagick-7" \
CPPFLAGS="$CPPFLAGS -I/usr/local/include/ImageMagick-7" \
RUSTFLAGS="-L/usr/lib/$TOOL -C link-arg=-Wl,-rpath-link,/usr/lib/$TOOL -L/usr/$TOOL/lib -C link-arg=-Wl,-rpath-link,/usr/$TOOL/lib"
PATH=$PATH:/opt/build/.cargo/bin
COPY --from=rust --chown=build:build /opt/build/.cargo /opt/build/.cargo
COPY --from=rust --chown=build:build /opt/build/.rustup /opt/build/.rustup
COPY --from=imagemagick-builder /usr/local/ /usr/local
COPY root/ /
USER build

2
docker/prod/docker-compose.yml

@ -2,7 +2,7 @@ version: '3.3'
services:
pictrs:
image: asonix/pictrs:v0.3.0-alpha.13-shell-out-r3
image: asonix/pictrs:v0.3.0-alpha.13-shell-out-r4
ports:
- "127.0.0.1:8080:8080"
restart: always

74
src/exiv2.rs

@ -3,26 +3,11 @@ pub(crate) enum Exvi2Error {
#[error("Failed to interface with exiv2")]
IO(#[from] std::io::Error),
#[error("Mime Parse: {0}")]
Mime(#[from] mime::FromStrError),
#[error("Identify semaphore is closed")]
Closed,
#[error("Requested information is not present")]
Missing,
#[error("Exiv2 command failed")]
Status,
#[error("Requested information was present, but not supported")]
Unsupported,
}
pub(crate) struct Details {
pub(crate) mime_type: mime::Mime,
pub(crate) width: usize,
pub(crate) height: usize,
}
static MAX_READS: once_cell::sync::OnceCell<tokio::sync::Semaphore> =
@ -54,67 +39,8 @@ where
Ok(())
}
pub(crate) async fn details<P>(file: P) -> Result<Details, Exvi2Error>
where
P: AsRef<std::path::Path>,
{
let permit = semaphore().acquire().await?;
let output = tokio::process::Command::new("exiv2")
.arg(&"pr")
.arg(&file.as_ref())
.output()
.await?;
drop(permit);
let s = String::from_utf8_lossy(&output.stdout);
parse_output(s)
}
fn parse_output(s: std::borrow::Cow<'_, str>) -> Result<Details, Exvi2Error> {
let mime_line = s
.lines()
.find(|line| line.starts_with("MIME"))
.ok_or_else(|| Exvi2Error::Missing)?;
let mut segments = mime_line.rsplit(':');
let mime_type = segments.next().ok_or_else(|| Exvi2Error::Missing)?.trim();
let resolution_line = s
.lines()
.find(|line| line.starts_with("Image size"))
.ok_or_else(|| Exvi2Error::Missing)?;
let mut segments = resolution_line.rsplit(':');
let resolution = segments.next().ok_or_else(|| Exvi2Error::Missing)?;
let mut resolution_segments = resolution.split('x');
let width_str = resolution_segments
.next()
.ok_or_else(|| Exvi2Error::Missing)?
.trim();
let height_str = resolution_segments
.next()
.ok_or_else(|| Exvi2Error::Missing)?
.trim();
let width = width_str.parse()?;
let height = height_str.parse()?;
Ok(Details {
mime_type: mime_type.parse()?,
width,
height,
})
}
impl From<tokio::sync::AcquireError> for Exvi2Error {
fn from(_: tokio::sync::AcquireError) -> Exvi2Error {
Exvi2Error::Closed
}
}
impl From<std::num::ParseIntError> for Exvi2Error {
fn from(_: std::num::ParseIntError) -> Exvi2Error {
Exvi2Error::Unsupported
}
}

71
src/magick.rs

@ -21,6 +21,12 @@ pub(crate) enum ValidInputType {
Webp,
}
pub(crate) struct Details {
pub(crate) mime_type: mime::Mime,
pub(crate) width: usize,
pub(crate) height: usize,
}
static MAX_CONVERSIONS: once_cell::sync::OnceCell<tokio::sync::Semaphore> =
once_cell::sync::OnceCell::new();
@ -61,6 +67,65 @@ where
Ok(())
}
pub(crate) async fn details<P>(file: P) -> Result<Details, MagickError>
where
P: AsRef<std::path::Path>,
{
let permit = semaphore().acquire().await?;
let output = tokio::process::Command::new("magick")
.args([&"identify", &"-ping", &"-format", &"%w %h | %m\n"])
.arg(&file.as_ref())
.output()
.await?;
drop(permit);
let s = String::from_utf8_lossy(&output.stdout);
tracing::debug!("lines: {}", s);
let mut lines = s.lines();
let first = lines.next().ok_or_else(|| MagickError::Format)?;
let mut segments = first.split('|');
let dimensions = segments.next().ok_or_else(|| MagickError::Format)?.trim();
tracing::debug!("dimensions: {}", dimensions);
let mut dims = dimensions.split(' ');
let width = dims
.next()
.ok_or_else(|| MagickError::Format)?
.trim()
.parse()?;
let height = dims
.next()
.ok_or_else(|| MagickError::Format)?
.trim()
.parse()?;
let format = segments.next().ok_or_else(|| MagickError::Format)?.trim();
tracing::debug!("format: {}", format);
if !lines.all(|item| item.ends_with(format)) {
return Err(MagickError::Format);
}
let mime_type = match format {
"MP4" => crate::validate::video_mp4(),
"GIF" => mime::IMAGE_GIF,
"PNG" => mime::IMAGE_PNG,
"JPEG" => mime::IMAGE_JPEG,
"WEBP" => crate::validate::image_webp(),
_ => return Err(MagickError::Format),
};
Ok(Details {
mime_type,
width,
height,
})
}
pub(crate) async fn input_type<P>(file: &P) -> Result<ValidInputType, MagickError>
where
P: AsRef<std::path::Path>,
@ -134,3 +199,9 @@ impl From<tokio::sync::AcquireError> for MagickError {
MagickError::Closed
}
}
impl From<std::num::ParseIntError> for MagickError {
fn from(_: std::num::ParseIntError) -> MagickError {
MagickError::Format
}
}

2
src/upload_manager.rs

@ -101,7 +101,7 @@ impl Details {
where
P: AsRef<std::path::Path>,
{
let details = crate::exiv2::details(&path).await?;
let details = crate::magick::details(&path).await?;
Ok(Details::now(
details.width,

Loading…
Cancel
Save