From 9a5d9fa8ade2b9a15ba5031d759c67bbe6c8d490 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Fri, 6 Nov 2026 08:06:51 -0600 Subject: [PATCH 1/5] K2GO-555 feat(deploy): tar -tv verbose line parser for byte-weighted extract Parse a GNU tar -tv line into member path + byte size. The path can contain spaces, so the parse anchors on the fixed date/time; link targets are stripped so the path matches what the extract pass prints. Dirs/symlinks/hardlinks count 0 bytes. Pure + unit-tested (samples verbatim from the bundled libtar 1.35 on device). First piece of byte-weighted rootfs extract progress. --- .../k2go/deploy/domain/TarVerboseLine.java | 76 +++++++++++++++++++ .../deploy/domain/TarVerboseLineTest.java | 71 +++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java create mode 100644 controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java diff --git a/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java b/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java new file mode 100644 index 000000000..4b3b1138f --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java @@ -0,0 +1,76 @@ +/* + * ============================================================================ + * Name : TarVerboseLine.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-555. Parse one `tar -tv` verbose listing line into its + * member path and byte size, so the rootfs Extract phase can + * weight progress by bytes written instead of by file count. + * File counts make the ETA absurd on slow phones, where rootfs + * members vary hugely in size. Pure JVM (no android.*) => + * unit-testable. + * ============================================================================ + */ +package org.appdevforall.k2go.deploy.domain; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * One line of GNU tar verbose output ({@code tar -tv}), as the bundled libtar (GNU tar 1.35) prints: + * 
MODE OWNER/GROUP SIZE YYYY-MM-DD HH:MM PATH[ -> TARGET]
+ * for example {@code -rw-rw-rw- shell/shell 45678 2026-11-06 08:00 ./sub/file_b.dat}. + * + *

The path can contain spaces, so the parse anchors on the fixed date and time fields rather than + * splitting on whitespace. Only a regular file carries its own data; a directory, symlink or hardlink + * reports (or is given) size 0, so it adds nothing to the byte total. The symlink/hardlink target is + * stripped from the path so it matches the plain path the extract pass ({@code tar -xv}) prints, which + * lets the extract look its size up by path. An unmatched line (blank, a stray warning, a device node + * whose "major,minor" is not a byte size) parses to {@code null}, which the caller counts as 0 bytes. + */ +public final class TarVerboseLine { + + // mode, owner/group, size, date, time, then the rest (the path, which may contain spaces). + private static final Pattern LINE = Pattern.compile( + "^(\\S+)\\s+\\S+\\s+(\\d+)\\s+\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}\\s+(.+)$"); + + /** The member path, exactly as the extract pass prints it (link target stripped). */ + public final String path; + /** Bytes of this member's own data: 0 for a directory, symlink or hardlink. */ + public final long size; + + private TarVerboseLine(String path, long size) { + this.path = path; + this.size = size; + } + + /** Parse one verbose listing line, or {@code null} when it is not a sizeable member line. */ + public static TarVerboseLine parse(String line) { + if (line == null) return null; + Matcher m = LINE.matcher(line); + if (!m.matches()) return null; + final char type = m.group(1).charAt(0); + long size; + try { + size = Long.parseLong(m.group(2)); + } catch (NumberFormatException e) { + return null; + } + String path = m.group(3); + if (type == 'l') { // symlink: "path -> target" + path = stripFrom(path, " -> "); + size = 0L; + } else if (type == 'h') { // hardlink: "path link to target" (data counted at the target) + path = stripFrom(path, " link to "); + size = 0L; + } else if (type != '-') { // directory, fifo, etc.: no data of its own + size = 0L; + } + return new TarVerboseLine(path, size); + } + + private static String stripFrom(String path, String sep) { + int i = path.indexOf(sep); + return i >= 0 ? path.substring(0, i) : path; + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java b/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java new file mode 100644 index 000000000..829fa2bfd --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java @@ -0,0 +1,71 @@ +package org.appdevforall.k2go.deploy.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +/** + * Unit tests for {@link TarVerboseLine} — parsing GNU tar -tv lines (bundled libtar 1.35) into + * path + byte size for byte-weighted extract progress (K2GO-555). The sample lines are verbatim + * from the bundled tar on device. + */ +public class TarVerboseLineTest { + + @Test + public void parsesRegularFileSize() { + TarVerboseLine e = TarVerboseLine.parse("-rw-rw-rw- shell/shell 45678 2026-11-06 08:00 ./sub/file_b.dat"); + assertEquals("./sub/file_b.dat", e.path); + assertEquals(45678L, e.size); + } + + @Test + public void parsesPathWithSpaces() { + TarVerboseLine e = TarVerboseLine.parse("-rw-rw-rw- shell/shell 7 2026-11-06 08:00 ./name with space.txt"); + assertEquals("./name with space.txt", e.path); + assertEquals(7L, e.size); + } + + @Test + public void parsesLargeSize() { + TarVerboseLine e = TarVerboseLine.parse("-rw-rw-rw- shell/shell 1048577 2026-11-06 08:00 ./big_over_1mb.bin"); + assertEquals(1048577L, e.size); + } + + @Test + public void directoryHasZeroSize() { + TarVerboseLine e = TarVerboseLine.parse("drwxrwxrwx shell/shell 0 2026-11-06 08:00 ./empty_dir/"); + assertEquals("./empty_dir/", e.path); + assertEquals(0L, e.size); + } + + @Test + public void symlinkStripsTargetAndCountsZero() { + TarVerboseLine e = TarVerboseLine.parse("lrwxrwxrwx shell/shell 0 2026-11-06 08:00 ./link_c -> file_a.txt"); + assertEquals("./link_c", e.path); // matches the plain path the extract pass prints + assertEquals(0L, e.size); + } + + @Test + public void hardlinkStripsTargetAndCountsZero() { + TarVerboseLine e = TarVerboseLine.parse("hrw-rw-rw- shell/shell 0 2026-11-06 08:00 ./hard_d.txt link to ./file_a.txt"); + assertEquals("./hard_d.txt", e.path); + assertEquals(0L, e.size); + } + + @Test + public void modeWithAclSuffixStillParses() { + // GNU tar may append '+' or '.' to the mode for an ACL / SELinux context; the type char is still first. + TarVerboseLine e = TarVerboseLine.parse("-rw-rw-rw-+ shell/shell 100 2026-11-06 08:00 ./acl_file"); + assertEquals("./acl_file", e.path); + assertEquals(100L, e.size); + } + + @Test + public void rejectsNonMemberLines() { + assertNull(TarVerboseLine.parse(null)); + assertNull(TarVerboseLine.parse("")); + assertNull(TarVerboseLine.parse("tar: some warning on stderr")); + assertNull(TarVerboseLine.parse("./plain-name-line")); // extract-pass style line, not a verbose listing + } +} From aaf40044537030f779f72b6f8eb98d77e52b02f5 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Fri, 6 Nov 2026 10:18:32 -0600 Subject: [PATCH 2/5] K2GO-555 fix(deploy): TarVerboseLine captures every member path, size for regular files only Device nodes show major,minor in the size column; the path-traversal and rootfs guards read every member, so the parser must not drop a member whose size field is not a byte count. Capture the path for all member types; parse a byte size only for a regular file. --- .../k2go/deploy/domain/TarVerboseLine.java | 53 ++++++++++++------- .../deploy/domain/TarVerboseLineTest.java | 17 ++++++ 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java b/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java index 4b3b1138f..43dceba1c 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/deploy/domain/TarVerboseLine.java @@ -21,22 +21,27 @@ * 

MODE OWNER/GROUP SIZE YYYY-MM-DD HH:MM PATH[ -> TARGET]
* for example {@code -rw-rw-rw- shell/shell 45678 2026-11-06 08:00 ./sub/file_b.dat}. * - *

The path can contain spaces, so the parse anchors on the fixed date and time fields rather than - * splitting on whitespace. Only a regular file carries its own data; a directory, symlink or hardlink - * reports (or is given) size 0, so it adds nothing to the byte total. The symlink/hardlink target is - * stripped from the path so it matches the plain path the extract pass ({@code tar -xv}) prints, which - * lets the extract look its size up by path. An unmatched line (blank, a stray warning, a device node - * whose "major,minor" is not a byte size) parses to {@code null}, which the caller counts as 0 bytes. + *

The parse anchors on the fixed date and time fields, not on whitespace, because the path can + * contain spaces. It always captures the path, even when the "size" column is not a byte count: a + * device node prints {@code major,minor} there, and a rootfs carries many device nodes under + * {@code ./dev}. The path-traversal and rootfs guards read these paths, so dropping a member whose + * size field is unusual would blind those guards. Only a regular file carries its own data; a + * directory, symlink, hardlink or device node contributes 0 bytes to the total. The symlink or + * hardlink target is stripped from the path so it matches the plain path the extract pass + * ({@code tar -xv}) prints, which lets the extract look its size up by path. A line that is not a + * member at all (blank, a stray warning) parses to {@code null}, which the caller counts as 0 bytes. */ public final class TarVerboseLine { - // mode, owner/group, size, date, time, then the rest (the path, which may contain spaces). + // mode, owner/group, size-field (any token: a byte count or a device "major,minor"), date, time + // (optional seconds), then the rest (the path, which may contain spaces). The size field is lazy + // so it stops at the first date-shaped token, which is the member mtime, never a date in the path. private static final Pattern LINE = Pattern.compile( - "^(\\S+)\\s+\\S+\\s+(\\d+)\\s+\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}\\s+(.+)$"); + "^(\\S+)\\s+\\S+\\s+(.+?)\\s+\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}(?::\\d{2})?\\s+(.+)$"); /** The member path, exactly as the extract pass prints it (link target stripped). */ public final String path; - /** Bytes of this member's own data: 0 for a directory, symlink or hardlink. */ + /** Bytes of this member's own data: 0 for a directory, symlink, hardlink or device node. */ public final long size; private TarVerboseLine(String path, long size) { @@ -44,31 +49,39 @@ private TarVerboseLine(String path, long size) { this.size = size; } - /** Parse one verbose listing line, or {@code null} when it is not a sizeable member line. */ + /** Parse one verbose listing line, or {@code null} when it is not a member line. */ public static TarVerboseLine parse(String line) { if (line == null) return null; Matcher m = LINE.matcher(line); if (!m.matches()) return null; final char type = m.group(1).charAt(0); - long size; - try { - size = Long.parseLong(m.group(2)); - } catch (NumberFormatException e) { - return null; - } String path = m.group(3); + long size = 0L; if (type == 'l') { // symlink: "path -> target" path = stripFrom(path, " -> "); - size = 0L; } else if (type == 'h') { // hardlink: "path link to target" (data counted at the target) path = stripFrom(path, " link to "); - size = 0L; - } else if (type != '-') { // directory, fifo, etc.: no data of its own - size = 0L; + } else if (type == '-') { // regular file: the only type that carries its own data + size = parseSize(m.group(2)); } + // directory, device node, fifo, socket: path kept for the guards, size stays 0. return new TarVerboseLine(path, size); } + /** A regular file's size is a plain decimal; anything else (a device "major,minor") is 0 bytes. */ + private static long parseSize(String field) { + if (field.isEmpty()) return 0L; + for (int i = 0; i < field.length(); i++) { + char c = field.charAt(i); + if (c < '0' || c > '9') return 0L; + } + try { + return Long.parseLong(field); + } catch (NumberFormatException e) { + return 0L; + } + } + private static String stripFrom(String path, String sep) { int i = path.indexOf(sep); return i >= 0 ? path.substring(0, i) : path; diff --git a/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java b/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java index 829fa2bfd..1ac19f348 100644 --- a/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java +++ b/controller/app/src/test/java/org/appdevforall/k2go/deploy/domain/TarVerboseLineTest.java @@ -53,6 +53,23 @@ public void hardlinkStripsTargetAndCountsZero() { assertEquals(0L, e.size); } + @Test + public void deviceNodeKeepsPathZeroSize() { + // A char/block device shows "major,minor" where a file shows its size. The path must still be + // captured: a rootfs has many nodes under ./dev and the traversal guard checks every member. + TarVerboseLine e = TarVerboseLine.parse("crw-rw-rw- root/root 1,3 2026-11-06 08:00 ./dev/null"); + assertEquals("./dev/null", e.path); + assertEquals(0L, e.size); + } + + @Test + public void deviceNodeWithSpaceInMajorMinor() { + // Some tar builds print "major, minor" with a space; the path is still captured, size 0. + TarVerboseLine e = TarVerboseLine.parse("brw-rw-rw- root/root 8, 0 2026-11-06 08:00 ./dev/sda"); + assertEquals("./dev/sda", e.path); + assertEquals(0L, e.size); + } + @Test public void modeWithAclSuffixStillParses() { // GNU tar may append '+' or '.' to the mode for an ACL / SELinux context; the type char is still first. From a22ddebf85a03169cbf16b1f1e3f6202371cf9c9 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Fri, 6 Nov 2026 10:20:07 -0600 Subject: [PATCH 3/5] K2GO-555 feat(system): WindowedRate, a shared trailing-window transfer rate A rate over the last window, not the cumulative average from the start. The cumulative average reads too low after a slow start and lags a recovery, which is what made the rootfs extract ETA read absurd minutes. Content-exchange and the extract pass will share this one pure, unit-tested helper. --- .../k2go/system/domain/WindowedRate.java | 67 +++++++++++++++++++ .../k2go/system/domain/WindowedRateTest.java | 59 ++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/system/domain/WindowedRate.java create mode 100644 controller/app/src/test/java/org/appdevforall/k2go/system/domain/WindowedRateTest.java diff --git a/controller/app/src/main/java/org/appdevforall/k2go/system/domain/WindowedRate.java b/controller/app/src/main/java/org/appdevforall/k2go/system/domain/WindowedRate.java new file mode 100644 index 000000000..9dfcaf02d --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/system/domain/WindowedRate.java @@ -0,0 +1,67 @@ +/* + * ============================================================================ + * Name : WindowedRate.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-555. A trailing-window transfer rate: bytes per second over + * the last window, not the cumulative average from the start. + * The content-exchange copy loops and the rootfs extract pass share + * this one implementation. Pure JVM (no android.*) => unit-testable. + * ============================================================================ + */ +package org.appdevforall.k2go.system.domain; + +/** + * The recent transfer rate, measured over a trailing time window. + * + *

{@link TransferRate} averages over the whole run, which is right when reports are coarse and a + * stable number matters more than reacting to the last second. This is the opposite choice: a byte + * stream that warms up slowly, or speeds up and slows down, needs a rate that follows the current + * throughput. The cumulative average reads far too low after a slow start and then lags a recovery, + * which is exactly what made the rootfs extract ETA read absurd minutes on a slow phone. + * + *

It anchors at the first sample, and re-anchors once the window is wide enough, reporting the + * bytes moved since the last anchor divided by the elapsed time. Between re-anchors it holds the last + * window's rate, so the shown number is steady rather than flickering on every tick. It is stateful, + * so each transfer owns one instance; a single caller thread feeds it, so it needs no synchronization. + * + *

Pure and unit-tested: the caller supplies the clock reading, so a test drives it with a fake one. + */ +public final class WindowedRate { + + private final long windowMs; + private long anchorMs; + private long anchorBytes; + private long rate = 0L; // last closed window's bytes/sec; 0 means "nothing to report yet" + private boolean started = false; + + /** @param windowMs the trailing window width; a value below 1 is treated as 1 ms. */ + public WindowedRate(long windowMs) { + this.windowMs = windowMs < 1L ? 1L : windowMs; + } + + /** + * Feed the cumulative bytes transferred so far and the current monotonic clock reading (ms). + * Returns the current rate in bytes per second: {@code 0} until the first window closes, then the + * most recent window's throughput, held until the next window closes. + * + *

{@code 0} is the "say nothing" value, consistent with {@link TransferRate}: a caller omits + * the rate and treats the ETA as not-yet-estimable rather than printing a stall. + */ + public long sample(long cumulativeBytes, long nowMs) { + if (!started) { + anchorMs = nowMs; + anchorBytes = cumulativeBytes; + started = true; + return rate; + } + long dt = nowMs - anchorMs; + if (dt >= windowMs) { + long db = cumulativeBytes - anchorBytes; + rate = db > 0L ? db * 1000L / dt : 0L; + anchorMs = nowMs; + anchorBytes = cumulativeBytes; + } + return rate; + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/system/domain/WindowedRateTest.java b/controller/app/src/test/java/org/appdevforall/k2go/system/domain/WindowedRateTest.java new file mode 100644 index 000000000..a75bbc1c3 --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/system/domain/WindowedRateTest.java @@ -0,0 +1,59 @@ +package org.appdevforall.k2go.system.domain; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Unit tests for {@link WindowedRate} — the trailing-window transfer rate (K2GO-555). The tests drive + * a fake monotonic clock so they assert exact rates without real time. + */ +public class WindowedRateTest { + + @Test + public void reportsNothingBeforeTheFirstWindowCloses() { + WindowedRate r = new WindowedRate(500L); + assertEquals(0L, r.sample(0L, 0L)); // first sample anchors + assertEquals(0L, r.sample(1_000L, 200L)); // window not wide enough yet + } + + @Test + public void reportsRecentThroughputOnceWindowCloses() { + WindowedRate r = new WindowedRate(500L); + r.sample(0L, 0L); + // 500 KB over 500 ms = 1 MB/s. + assertEquals(1_000_000L, r.sample(500_000L, 500L)); + } + + @Test + public void holdsLastRateBetweenWindows() { + WindowedRate r = new WindowedRate(500L); + r.sample(0L, 0L); + long first = r.sample(500_000L, 500L); // window closes, rate = 1 MB/s + assertEquals(first, r.sample(500_050L, 600L)); // 100 ms later: same held rate, window not closed + } + + @Test + public void followsASpeedChangeOnTheNextWindow() { + WindowedRate r = new WindowedRate(500L); + r.sample(0L, 0L); + assertEquals(1_000_000L, r.sample(500_000L, 500L)); // fast window: 1 MB/s + // Next window moves only 50 KB over 500 ms = 100 KB/s. The cumulative average would still read + // ~0.5 MB/s here; the window reflects the slowdown. + assertEquals(100_000L, r.sample(550_000L, 1_000L)); + } + + @Test + public void reportsZeroForAStalledWindow() { + WindowedRate r = new WindowedRate(500L); + r.sample(1_000L, 0L); + assertEquals(0L, r.sample(1_000L, 500L)); // no bytes moved across the window + } + + @Test + public void nonPositiveWindowDoesNotDivideByZero() { + WindowedRate r = new WindowedRate(0L); // clamped to 1 ms + r.sample(0L, 0L); + assertEquals(1_000_000L, r.sample(1_000L, 1L)); // 1000 B over 1 ms = 1 MB/s + } +} From 1c8d0dbd4ba13fcb0c805c3ee7bf74e590130d43 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Fri, 6 Nov 2026 10:25:07 -0600 Subject: [PATCH 4/5] K2GO-555 fix(deploy): byte-weighted rootfs extract progress and ETA List with tar -tv and keep each member size, so the extract bar and ETA are weighted by uncompressed bytes written, not file count. Members vary hugely in size, so the file-count ETA climbed to absurd values mid-extract on a slow phone and a healthy install looked hung. The extract rate is a trailing window, not the cumulative average from the pass start. Byte weighting degrades to file count when the listing yields no sizes, so the bar can never stall at zero. Ripple: the traversal and rootfs guards now get parsed paths, not the raw tar -t lines they used before; the rootfs validator extracts a probe member by that path, so it must be the clean path. VERIFY still counts compressed bytes read. --- .../org/appdevforall/k2go/TarExtractor.java | 128 +++++++++++++----- 1 file changed, 92 insertions(+), 36 deletions(-) diff --git a/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java b/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java index d84d22b2f..f7b09a9cd 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java @@ -11,6 +11,9 @@ import org.appdevforall.k2go.deploy.domain.ArchiveEntry; import org.appdevforall.k2go.deploy.domain.ExtractProgress; +import org.appdevforall.k2go.deploy.domain.TarVerboseLine; +import org.appdevforall.k2go.system.domain.TransferRate; +import org.appdevforall.k2go.system.domain.WindowedRate; import android.content.Context; import android.os.Handler; @@ -24,7 +27,9 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.zip.GZIPInputStream; public class TarExtractor { @@ -32,6 +37,24 @@ public class TarExtractor { private Process tarProcess; private boolean isExtracting = false; + /** K2GO-555: a wide trailing window for the extract rate. tar names a member at its start, so the + * per-member byte delta is bursty; a wide window averages the burst into honest throughput and + * pairs with the presentation EtaSmoother's dwell. */ + private static final long EXTRACT_RATE_WINDOW_MS = 5_000L; + + /** K2GO-555: the archive listing -- member paths (for the traversal and rootfs guards) plus the + * per-path uncompressed size and their total, so the extract phase can weight progress by bytes. */ + private static final class Listing { + final List paths; + final Map sizeByPath; + final long totalBytes; + Listing(List paths, Map sizeByPath, long totalBytes) { + this.paths = paths; + this.sizeByPath = sizeByPath; + this.totalBytes = totalBytes; + } + } + /** ADFA-5118: the two passes the app makes over the same .tar.gz — the safety/listing pass * ({@code VERIFY}, {@link #listEntries}) and the extraction pass ({@code EXTRACT}). */ public enum Phase { VERIFY, EXTRACT } @@ -185,17 +208,21 @@ public void startExtraction(Context context, String archivePath, String destDir, } } - List entries = listEntries(tarBinary, archivePath, isGzip, cancelledBeforeExtract, pauseAtBoundary, listener); - for (String entry : entries) { + Listing listing = listEntries(tarBinary, archivePath, isGzip, cancelledBeforeExtract, pauseAtBoundary, listener); + for (String entry : listing.paths) { if (ArchiveEntry.escapesRoot(entry)) { throw new Exception("Unsafe archive entry (path traversal): " + entry); } } - // ADFA-4915: entries.size() is the extraction denominator (members), already - // computed above for the traversal/ABI checks — no extra tar pass. - final long totalMembers = entries.size(); + // ADFA-4915 / K2GO-555: the extraction denominator, from the one listing pass (no extra + // tar pass). totalBytes (uncompressed) weights progress by real work; totalMembers is the + // fallback when the listing yields no sizes, so the bar can never stall at zero. + final long totalMembers = listing.paths.size(); + final long totalBytes = listing.totalBytes; + final boolean byteMode = totalBytes > 0L; final long[] doneMembers = {0L}; + final long[] doneBytes = {0L}; // For untrusted imports/restores: it must be a valid rootfs of THIS // app's architecture (ABI policy: 32<->32, 64<->64). Reuses the @@ -203,7 +230,7 @@ public void startExtraction(Context context, String archivePath, String destDir, if (validateRootfs) { org.appdevforall.k2go.deploy.data.RootfsArchiveValidator.Result vr = org.appdevforall.k2go.deploy.data.RootfsArchiveValidator - .validateWithEntries(context, archivePath, isGzip, tarBinary, entries); + .validateWithEntries(context, archivePath, isGzip, tarBinary, listing.paths); String why = org.appdevforall.k2go.deploy.data.RootfsArchiveValidator .rejectionMessage(context, vr); if (why != null) { @@ -249,6 +276,9 @@ public void startExtraction(Context context, String archivePath, String destDir, // reader thread below (tar -v emits one line per member as it lands on disk). extractStartMs // is the pass clock that thread uses for the rate/ETA. final long extractStartMs = SystemClock.elapsedRealtime(); + // K2GO-555: the EXTRACT rate is a trailing window, not the cumulative average. The reader + // thread is the only writer, so this instance needs no synchronization. + final WindowedRate extractRate = new WindowedRate(EXTRACT_RATE_WINDOW_MS); Thread readerThread = new Thread(() -> { long[] lastEmit = {0L}; long[] lastLog = {0L}; @@ -266,23 +296,25 @@ public void startExtraction(Context context, String archivePath, String destDir, Log.d(TAG, "Tar Output: " + line); } doneMembers[0]++; // ADFA-4915: ~one verbose line per member (stray stderr lines just push toward the 99% clamp) + // K2GO-555: weight the EXTRACT bar + ETA by uncompressed bytes written, not + // member count. tar -v names each member as it lands on disk; look its size up + // from the listing and accumulate. An unknown line (stray/stderr) adds 0 bytes, + // so there is no index drift. Members vary hugely in size, so the file-count ETA + // climbed to absurd values mid-extract; bytes track the real work. + Long sz = listing.sizeByPath.get(line); + if (sz != null) doneBytes[0] += sz; if (now - lastEmit[0] >= 50) { lastEmit[0] = now; final String l = line; - final long d = doneMembers[0]; - final int pct = ExtractProgress.percent(d, totalMembers); + final long d = byteMode ? doneBytes[0] : doneMembers[0]; + final long tot = byteMode ? totalBytes : totalMembers; + final int pct = ExtractProgress.percent(d, tot); + final long rate = extractRate.sample(d, SystemClock.elapsedRealtime() - extractStartMs); uiHandler.post(() -> { listener.onProgress(l); - listener.onProgress(pct, d, totalMembers, l); + listener.onProgress(pct, d, tot, l); }); - // K2GO-384: the unified EXTRACT bar + ETA track MEMBERS WRITTEN (one tar -v - // line per member as it lands on disk), not compressed bytes read from the - // .gz. The byte count saturated at ~99% the instant the archive was fully - // piped into tar, freezing the bar and reading "under a min" for minutes - // while tar was still writing the (much larger) rootfs. Members track the - // real work, so the bar keeps moving and the ETA reads honest minutes. - emitPhase(listener, uiHandler, Phase.EXTRACT, - d, totalMembers, SystemClock.elapsedRealtime() - extractStartMs, l); + emitPhase(listener, uiHandler, Phase.EXTRACT, d, tot, rate, l); } } } catch (Exception ignored) { @@ -395,15 +427,17 @@ public void startExtraction(Context context, String archivePath, String destDir, /** - * ADFA-5118: compute this pass's percent + a live ETA (rate averaged by TransferRate over the - * pass so far) and hand them to the listener on the UI thread. Used by both the VERIFY and the - * EXTRACT feeder; the unified 0..100 mapping is done by the presentation layer. + * ADFA-5118 / K2GO-555: compute this pass's percent and a live ETA from a caller-supplied rate, + * then hand them to the listener on the UI thread. The caller owns the rate source: VERIFY passes + * the cumulative average (reading the archive is steady work), EXTRACT passes a trailing-window + * rate (tar names a member at its start, so the per-member byte delta is bursty). The unified + * 0..100 mapping is done by the presentation layer. {@code ratePerSec} is in the same unit as + * {@code done}/{@code total} per second (bytes, or members on the no-size fallback). */ private static void emitPhase(ExtractionListener listener, Handler uiHandler, Phase phase, - long done, long total, long elapsedMs, String line) { - final long rate = org.appdevforall.k2go.system.domain.TransferRate.perSecond(done, elapsedMs); + long done, long total, long ratePerSec, String line) { final int pct = ExtractProgress.percent(done, total); - final long eta = ExtractProgress.etaSeconds(done, total, rate); + final long eta = ExtractProgress.etaSeconds(done, total, ratePerSec); final String l = line == null ? "" : line; uiHandler.post(() -> listener.onExtractPhase(phase, pct, eta, l)); } @@ -415,22 +449,29 @@ private static void emitPhase(ExtractionListener listener, Handler uiHandler, Ph * *

ADFA-5118: this is the VERIFY pass of the unified bar. On the gzip path it counts * compressed bytes consumed against the archive size on disk and emits determinate - * progress + an ETA (via {@code onExtractPhase}); {@code tar -t} already streams each - * member name, so the current file is surfaced too — symmetric with the extract pass. + * progress + an ETA (via {@code onExtractPhase}); {@code tar -tv} already streams each + * member, so the current file is surfaced too — symmetric with the extract pass. + * + *

K2GO-555: the listing is {@code tar -tv} (verbose), so each member carries its byte + * size. {@link TarVerboseLine} parses every line into a path (for the traversal and rootfs + * guards) plus the size, and the sizes feed the byte-weighted extract bar. */ - private List listEntries(String tarBinary, String archivePath, boolean isGzip, - java.util.concurrent.atomic.AtomicBoolean cancelledBeforeExtract, - java.util.concurrent.atomic.AtomicBoolean pauseAtBoundary, - ExtractionListener listener) throws Exception { + private Listing listEntries(String tarBinary, String archivePath, boolean isGzip, + java.util.concurrent.atomic.AtomicBoolean cancelledBeforeExtract, + java.util.concurrent.atomic.AtomicBoolean pauseAtBoundary, + ExtractionListener listener) throws Exception { List names = new ArrayList<>(); + Map sizeByPath = new LinkedHashMap<>(); + long[] totalBytes = {0L}; List listCmd = new ArrayList<>(); listCmd.add(tarBinary); if (isGzip) { - listCmd.add("-t"); + // K2GO-555: -v so each member line carries its size; gzip is decompressed in Java and piped. + listCmd.add("-tv"); listCmd.add("-f"); listCmd.add("-"); } else { - listCmd.add("-tf"); + listCmd.add("-tvf"); listCmd.add(archivePath); } @@ -456,8 +497,8 @@ private List listEntries(String tarBinary, String archivePath, boolean i long now = SystemClock.elapsedRealtime(); if (compressedTotal > 0L && now - lastEmit >= 200) { lastEmit = now; - emitPhase(listener, uiHandler, Phase.VERIFY, - cis.count, compressedTotal, now - startMs, lastFile[0]); + emitPhase(listener, uiHandler, Phase.VERIFY, cis.count, compressedTotal, + TransferRate.perSecond(cis.count, now - startMs), lastFile[0]); } } os.flush(); @@ -482,8 +523,23 @@ private List listEntries(String tarBinary, String archivePath, boolean i listProcess.destroy(); throw new HeldForDecisionException(); } - names.add(line); - lastFile[0] = line; // ADFA-5118: the member tar -t is listing right now + if (line.trim().isEmpty()) { + continue; + } + // K2GO-555: stdout of `tar -tv` is member lines only (stderr is separate here). Parse + // each into its path and size. The path-traversal and rootfs guards read these paths, so + // a line we cannot parse is a path we cannot verify -> fail closed, do not extract blind. + TarVerboseLine e = TarVerboseLine.parse(line); + if (e == null) { + listProcess.destroy(); + throw new Exception("Unparseable archive listing line: " + line); + } + names.add(e.path); + if (e.size > 0L) { + sizeByPath.put(e.path, e.size); + totalBytes[0] += e.size; + } + lastFile[0] = e.path; // ADFA-5118: the member tar is listing right now } } @@ -493,7 +549,7 @@ private List listEntries(String tarBinary, String archivePath, boolean i // Could not verify the archive -> fail closed rather than extract blind. throw new Exception("Could not read archive listing for verification (tar exit " + exitCode + ")"); } - return names; + return new Listing(names, sizeByPath, totalBytes[0]); } /** Available bytes on the destination's filesystem, or -1 if unavailable (ADFA-4544). */ From f61e995152aae4ec0340ad83480d133b539b4880 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Fri, 6 Nov 2026 10:29:52 -0600 Subject: [PATCH 5/5] K2GO-555 docs(deploy): correct onProgress units note, drop '--' in javadoc Review pass: onProgress done/total are bytes in byteMode (advisory; consumers use percent); use ':' not '--' per CLAUDE.md. --- .../main/java/org/appdevforall/k2go/TarExtractor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java b/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java index f7b09a9cd..25815e1af 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/TarExtractor.java @@ -42,7 +42,7 @@ public class TarExtractor { * pairs with the presentation EtaSmoother's dwell. */ private static final long EXTRACT_RATE_WINDOW_MS = 5_000L; - /** K2GO-555: the archive listing -- member paths (for the traversal and rootfs guards) plus the + /** K2GO-555: the archive listing: member paths (for the traversal and rootfs guards) plus the * per-path uncompressed size and their total, so the extract phase can weight progress by bytes. */ private static final class Listing { final List paths; @@ -69,9 +69,10 @@ default void onProgress(String line) { } /** * ADFA-4915: determinate extraction progress. {@code percent} is clamped to - * [0,99] while extracting and set to 100 exactly once on completion; {@code done} - * / {@code total} are archive-member counts; {@code line} is the current verbose - * tar line (may be empty). Default no-op. + * [0,99] while extracting and set to 100 exactly once on completion. K2GO-555: + * {@code done} / {@code total} are uncompressed bytes when the listing has sizes, + * else archive-member counts; {@code line} is the current verbose tar line (may be + * empty). Consumers use {@code percent}; the counts are advisory. Default no-op. */ default void onProgress(int percent, long done, long total, String line) { }