🐳 build (ptth_relay): clean up Docker build process

The new method is much nicer and doesn't require the manual make-old-git
step. The top-level command is actually build_and_minimize.bash, which uses
`git archive` to unpack the last Git commit and build with _that_ Dockerfile
and Docker context. This is better for determinism. It's similar to our build
process for that one big project at work.
main
_ 2020-12-12 01:53:20 +00:00
parent 951fe27b5f
commit 0c5a37b441
10 changed files with 2305 additions and 96 deletions

1
.gitignore vendored
View File

@ -1,4 +1,3 @@
/Cargo.lock
/config /config
/*.tar.gz /*.tar.gz
/ptth_server.toml /ptth_server.toml

2235
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,40 +1,77 @@
FROM rust:1.47-slim-buster as build # https://whitfin.io/speeding-up-rust-docker-builds/
# TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the
WORKDIR /usr/src # 1.47 slim-buster
FROM rust@sha256:2a902de987345f126fe59daca200afae1fccb6f68e14e9a27c0fd9cf39f9743f as build
RUN apt-get update \ #RUN apk add libseccomp-dev
&& apt-get install -y git pkg-config libssl-dev
# Make sure the dependencies are all cached so we won't hammer crates.io WORKDIR /
ENV USER root
ADD old-git.tar.gz . # create empty shell projects
RUN git checkout 7925d9be95df600c84efd084ec77c81c0da3e651 \ RUN cargo new --bin ptth
&& git reset --hard \
&& cargo check -p ptth_relay
RUN cargo test --release --all \ WORKDIR /ptth
&& cargo build --release -p ptth_relay
COPY .git .git RUN \
cargo new --lib crates/always_equal && \
cargo new --lib crates/ptth_core && \
cargo new --bin crates/ptth_file_server_bin && \
cargo new --bin crates/ptth_relay && \
cargo new --bin crates/ptth_server
ARG gitcommithash=HEAD # copy over your manifests
COPY ./Cargo.lock ./
COPY ./Cargo.toml ./
COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/
COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/
COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/
COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/
COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/
RUN git checkout "$gitcommithash" \ # this build step will cache your dependencies
&& git reset --hard \ RUN cargo build --release -p ptth_relay
&& echo "pub const GIT_VERSION: Option <&str> = Some (\"$(git rev-parse HEAD)\");" > crates/ptth_relay/src/git_version.rs \
&& cargo test --release --all \
&& cargo build --release -p ptth_relay
FROM debian:buster-slim as deploy RUN \
rm \
src/*.rs \
crates/always_equal/src/*.rs \
crates/ptth_core/src/*.rs \
crates/ptth_file_server_bin/src/*.rs \
crates/ptth_relay/src/*.rs \
crates/ptth_server/src/*.rs
# Copy source tree
# Yes, I tried a few variations on the syntax. Dockerfiles are just rough.
COPY ./src/ ./src
COPY ./crates/ ./crates
COPY ./handlebars/ ./handlebars
# Bug in cargo's incremental build logic, triggered by
# Docker doing something funny with mtimes? Maybe?
RUN touch crates/ptth_core/src/lib.rs
ARG git_version
RUN echo -n "$git_version" > crates/ptth_relay/src/git_version.txt
# build for release
# gate only on ptth_relay tests for now
RUN \
cargo build --release -p ptth_relay && \
cargo test --release -p ptth_relay
# buster-slim
FROM debian@sha256:062bbd9a1a58c9c5b8fc9d83a206371127ef268cfcc65f1a01227c6faebdb212
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y libssl1.1 ca-certificates \ && apt-get install -y libssl1.1 ca-certificates \
&& apt-get upgrade -y && apt-get upgrade -y
COPY --from=build /usr/src/target/release/ptth_relay /root COPY --from=build /ptth/target/release/ptth_relay /root/
COPY --from=build /usr/src/crates/ptth_relay/src/git_version.rs /root/git_version.rs COPY --from=build /ptth/crates/ptth_relay/src/git_version.txt /root/
COPY --from=build /usr/src/handlebars /root/handlebars COPY --from=build /ptth/handlebars /root/handlebars
WORKDIR /root WORKDIR /root
ENTRYPOINT ["./ptth_relay"]
CMD ["./ptth_relay"]

View File

@ -10,15 +10,17 @@ set -euo pipefail
TEMP_GIBBERISH="ptth_build_L6KLMVS6" TEMP_GIBBERISH="ptth_build_L6KLMVS6"
TEMP_TAR="$TEMP_GIBBERISH/ptth.tar" TEMP_TAR="$TEMP_GIBBERISH/ptth.tar"
UPLOADABLE_TAR="$PWD/ptth_latest.tar.gz" UPLOADABLE_TAR="$PWD/ptth_latest.tar.gz"
GIT_COMMITISH=$(git rev-parse main)
# This is magic and will need to be updated whenever we update the # This is magic and will need to be updated whenever we update the
# Debian layer. # Debian layer.
BOTTOM_LAYER="cec906613726ec32de92af0ec1cd6692c34df78782227f4415cd12c47a264dd4" BOTTOM_LAYER="cec906613726ec32de92af0ec1cd6692c34df78782227f4415cd12c47a264dd4"
rm -rf "$TEMP_GIBBERISH"
mkdir -p "$TEMP_GIBBERISH/ptth" mkdir -p "$TEMP_GIBBERISH/ptth"
sudo docker build -t ptth:latest . git archive --format=tar "$GIT_COMMITISH" | sudo docker build -t ptth:latest --build-arg "git_version=$GIT_COMMITISH" -
sudo docker image save ptth:latest | pv > "$TEMP_TAR" sudo docker image save ptth:latest | pv > "$TEMP_TAR"
tar -C "$TEMP_GIBBERISH/ptth" -xf "$TEMP_TAR" tar -C "$TEMP_GIBBERISH/ptth" -xf "$TEMP_TAR"

View File

@ -1 +1 @@
pub const GIT_VERSION: Option <&str> = None; pub const GIT_VERSION: &str = include_str! ("git_version.txt");

View File

@ -0,0 +1 @@
(Unknown)

View File

@ -32,7 +32,7 @@ stronger is ready.
## Proposed impl plan ## Proposed impl plan
- (X) Add feature flags to ptth_relay.toml for dev mode and scrapers - (X) Add feature flags to ptth_relay.toml for dev mode and scrapers
- ( ) Make sure Docker release CAN build - (X) Make sure Docker release CAN build
- ( ) Add failing test to block releases - ( ) Add failing test to block releases
- ( ) Make sure `cargo test` fails and Docker release can NOT build - ( ) Make sure `cargo test` fails and Docker release can NOT build
- ( ) Add hard-coded hash of 1 API key, with 1 week expiration - ( ) Add hard-coded hash of 1 API key, with 1 week expiration

View File

@ -1 +0,0 @@
tar -czf old-git.tar.gz .git

View File

@ -1,67 +0,0 @@
# https://whitfin.io/speeding-up-rust-docker-builds/
# TODO: https://stackoverflow.com/questions/57389547/how-to-define-the-context-for-a-docker-build-as-a-specific-commit-on-one-of-the
FROM rust:1.47-slim-buster as build
#RUN apk add libseccomp-dev
WORKDIR /
ENV USER root
# create empty shell projects
RUN cargo new --bin ptth
WORKDIR /ptth
RUN \
cargo new --lib crates/always_equal && \
cargo new --lib crates/ptth_core && \
cargo new --bin crates/ptth_file_server_bin && \
cargo new --bin crates/ptth_relay && \
cargo new --bin crates/ptth_server
# copy over your manifests
COPY ./Cargo.lock ./Cargo.lock
COPY ./Cargo.toml ./Cargo.toml
COPY ./crates/always_equal/Cargo.toml ./crates/always_equal/Cargo.toml
COPY ./crates/ptth_core/Cargo.toml ./crates/ptth_core/Cargo.toml
COPY ./crates/ptth_relay/Cargo.toml ./crates/ptth_relay/Cargo.toml
COPY ./crates/ptth_file_server_bin/Cargo.toml ./crates/ptth_file_server_bin/Cargo.toml
COPY ./crates/ptth_server/Cargo.toml ./crates/ptth_server/Cargo.toml
# this build step will cache your dependencies
RUN cargo build --release -p ptth_relay
RUN \
rm \
src/*.rs \
crates/always_equal/src/*.rs \
crates/ptth_core/src/*.rs \
crates/ptth_file_server_bin/src/*.rs \
crates/ptth_relay/src/*.rs \
crates/ptth_server/src/*.rs
# copy source tree
COPY ./src ./src
COPY ./crates ./crates
COPY ./handlebars ./handlebars
# Bug in cargo's incremental build logic, triggered by
# Docker doing something funny with mtimes? Maybe?
RUN touch crates/ptth_core/src/lib.rs
# build for release
RUN cargo build --release -p ptth_relay
FROM debian:buster-slim
RUN apt-get update \
&& apt-get install -y libssl1.1 ca-certificates \
&& apt-get upgrade -y
COPY --from=build /ptth/target/release/ptth_relay /root/ptth_relay
COPY --from=build /ptth/crates/ptth_relay/src/git_version.rs /root/git_version.rs
COPY --from=build /ptth/handlebars /root/handlebars
WORKDIR /root
ENTRYPOINT ["./ptth_relay"]

3
run_docker_image.bash Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
sudo docker run -it -v $PWD/config:/root/config -e RUST_LOG=ptth=trace ptth:latest