Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package org.breedinginsight.services.geno.impl;

import htsjdk.samtools.util.RuntimeIOException;
import htsjdk.samtools.util.SortingCollection;

import java.io.*;
import java.nio.charset.StandardCharsets;

/** Length-prefixed positional keys for HTSJDK's disk-backed sort. */
final class VcfPositionCodec implements SortingCollection.Codec<String> {
private DataInputStream input;
private DataOutputStream output;

@Override
public void setInputStream(InputStream stream) {
input = new DataInputStream(stream);
}

@Override
public void setOutputStream(OutputStream stream) {
output = new DataOutputStream(stream);
}

@Override
public void encode(String key) {
try {
byte[] bytes = key.getBytes(StandardCharsets.UTF_8);
output.writeInt(bytes.length);
output.write(bytes);
} catch (IOException e) {
throw new RuntimeIOException(e);
}
}

@Override
public String decode() {
try {
int first = input.read();
if (first == -1) {
return null;
}
int length = (first << 24) | (input.readUnsignedByte() << 16)
| (input.readUnsignedByte() << 8) | input.readUnsignedByte();
byte[] bytes = new byte[length];
input.readFully(bytes);
return new String(bytes, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeIOException(e);
}
}

@Override
public VcfPositionCodec clone() {
return new VcfPositionCodec();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import javax.inject.Singleton;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Path;

@Singleton
public class MimeTypeParser {
Expand Down Expand Up @@ -68,4 +69,12 @@ public MimeType getMimeType(byte[] fileContents, String filename) throws IOExcep
.getRegisteredMimeType(mediaType.toString());
}

public MimeType getMimeType(Path file, String filename) throws IOException, MimeTypeException {
Metadata metadata = new Metadata();
metadata.add(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
try (TikaInputStream stream = TikaInputStream.get(file)) {
return config.getMimeRepository().getRegisteredMimeType(detector.detect(stream, metadata).toString());
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.breedinginsight.services.geno.impl;

import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;

class GenotypeMultipartUploadTest {
@TempDir
Path directory;

@Test
void failedPartOrCompletionAbortsUploadAndPreservesOriginalError() throws Exception {
for (boolean failPart : new boolean[]{true, false}) {
S3Client s3 = mock(S3Client.class);
SimpleStorageService storage = mock(SimpleStorageService.class);
when(storage.getDefaultBucketName()).thenReturn("genotype");
when(s3.createMultipartUpload(any(CreateMultipartUploadRequest.class)))
.thenReturn(CreateMultipartUploadResponse.builder().uploadId("upload-id").build());
RuntimeException failure = new RuntimeException("Storage failed");
if (failPart) {
when(s3.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))).thenThrow(failure);
} else {
when(s3.uploadPart(any(UploadPartRequest.class), any(RequestBody.class)))
.thenReturn(UploadPartResponse.builder().eTag("etag").build());
when(s3.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenThrow(failure);
}
RuntimeException abortFailure = new RuntimeException("Abort failed");
when(s3.abortMultipartUpload(any(AbortMultipartUploadRequest.class))).thenThrow(abortFailure);
GigwaGenotypeServiceImpl service = new GigwaGenotypeServiceImpl(
"http://localhost/", "", "", "", null, null, null, null, null, null, null,
storage, s3, null, null, null, null, null);
Path file = Files.writeString(directory.resolve("sample.vcf"), "sample VCF");
assertSame(failure, assertThrows(RuntimeException.class,
() -> service.storeMultipartFile("key", file, Collections.emptyMap())));
assertArrayEquals(new Throwable[]{abortFailure}, failure.getSuppressed());
verify(s3).abortMultipartUpload(argThat((AbortMultipartUploadRequest request) ->
request.bucket().equals("genotype") && request.key().equals("key")
&& request.uploadId().equals("upload-id")));
if (failPart) {
verify(s3, never()).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,60 @@ public void testUpload() throws ApiException, AuthorizationException {
}
}

@Test
public void testMultipartUploadStreamsAllPartsAndAbortsOnReadFailure() throws Exception {
java.nio.file.Path file = java.nio.file.Files.createTempFile("vcf-multipart-test-", ".vcf");
String key = "stream-test/" + UUID.randomUUID() + ".vcf";
String failedKey = key + "-failed";
if (!storageService.listBucketNames().contains(storageService.getDefaultBucketName())) {
storageService.createBucket();
}
try {
byte[] chunk = new byte[1024 * 1024];
new Random(2970).nextBytes(chunk);
try (java.io.OutputStream output = java.nio.file.Files.newOutputStream(file)) {
for (int i = 0; i < 9; i++) {
output.write(chunk);
}
output.write(42);
}
String location = gigwaGenoStorageService.storeMultipartFile(key, file, Map.of("originalFileName", "large.vcf"));
assertNotNull(location);
assertTrue(location.contains(key));
try (java.io.InputStream stored = s3Client.getObject(
software.amazon.awssdk.services.s3.model.GetObjectRequest.builder()
.bucket(storageService.getDefaultBucketName()).key(key).build())) {
for (int i = 0; i < 9; i++) {
assertArrayEquals(chunk, stored.readNBytes(chunk.length));
}
assertEquals(42, stored.read());
assertEquals(-1, stored.read());
}
java.nio.file.Files.delete(file);
assertThrows(IOException.class, () ->
gigwaGenoStorageService.storeMultipartFile(failedKey, file, Collections.emptyMap()));
assertTrue(s3Client.listMultipartUploads(
software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest.builder()
.bucket(storageService.getDefaultBucketName()).prefix(failedKey).build())
.uploads().isEmpty(), "Failed upload must be aborted");
} finally {
java.nio.file.Files.deleteIfExists(file);
s3Client.deleteObject(software.amazon.awssdk.services.s3.model.DeleteObjectRequest.builder()
.bucket(storageService.getDefaultBucketName()).key(key).build());
}
}

@Test
public void testSubmitMissingHeader() throws Exception {
UUID programId = UUID.randomUUID();
UUID submissionId = UUID.randomUUID();
setupMocksForSubmitGenoData(programId, submissionId, Collections.emptyList());
ImportResponse response = submitGenoData(programId, "MISSINGHEADER", submissionId, "sample_missing_header.vcf");
assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
assertEquals("Could not find header row in file", response.getProgress().getMessage());
verify(genotypeImportDAO, never()).createGenotypeImportLink(eq(submissionId), any(), any());
}

@Test
public void testFetchGermplasmGenotype() throws AuthorizationException, ApiException, DoesNotExistException {
UUID programId = UUID.fromString("8b667063-480b-4b0a-862c-7eaa651dda28");
Expand Down Expand Up @@ -785,7 +839,7 @@ private void uploadGenoData(UUID programId, String programKey, UUID submissionId

System.out.println("====================== program ID: " + program.getId() + " ===============");
System.out.println("=================== submission ID: " + submissionId + " ===============");
gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, submissionId, new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), "sample.vcf", importUpload, progress);
gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, submissionId, java.nio.file.Path.of("src/test/resources/files/geno/sample.vcf"), "sample.vcf", importUpload, progress);
}

private ImportResponse submitGenoData(UUID programId, String programKey, UUID submissionId, String file)
Expand Down Expand Up @@ -827,7 +881,12 @@ private ImportResponse submitGenoData(UUID programId, String programKey, UUID su
user.getId(),
programId,
submissionId,
new TestFileUpload("src/test/resources/files/geno/" + file, MediaType.of("application/vcard"))
new TestFileUpload("src/test/resources/files/geno/" + file, MediaType.of("application/vcard")) {
@Override
public byte[] getBytes() {
throw new AssertionError("Genotype imports must stream the upload");
}
}
);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package org.breedinginsight.services.geno.impl;

import htsjdk.samtools.util.CloseableIterator;
import htsjdk.samtools.util.SortingCollection;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;

import static org.junit.jupiter.api.Assertions.*;

class VcfPositionCodecTest {
@TempDir
Path directory;

@Test
void duplicateKeysSurviveDiskSpillAndFilesAreCleanedUp() throws Exception {
SortingCollection<String> keys = SortingCollection.newInstance(
String.class, new VcfPositionCodec(), Comparator.naturalOrder(), 2, directory);
try {
keys.add("SNP:chr2:1");
keys.add("SNP:chr1:1");
keys.add("INDEL:chr1:1");
keys.add("SNP:chr1:1");
keys.doneAdding();
try (CloseableIterator<String> sorted = keys.iterator()) {
assertEquals("INDEL:chr1:1", sorted.next());
assertEquals("SNP:chr1:1", sorted.next());
assertEquals("SNP:chr1:1", sorted.next());
assertEquals("SNP:chr2:1", sorted.next());
assertFalse(sorted.hasNext());
}
} finally {
keys.cleanup();
}
try (java.util.stream.Stream<Path> files = Files.list(directory)) {
assertEquals(0, files.count());
}
}
}
2 changes: 2 additions & 0 deletions src/test/resources/files/geno/sample_missing_header.vcf
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
##fileformat=VCFv4.2
##source=BI-2970-regression
Loading