Downport fix commit (#1080)

(cherry picked from commit 94d6ad1)

bom/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.twelvemonkeys</groupId>
         <artifactId>twelvemonkeys</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
 
     <groupId>com.twelvemonkeys.bom</groupId>

common/common-image/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.common</groupId>
         <artifactId>common</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>common-image</artifactId>
     <packaging>jar</packaging>

common/common-io/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.common</groupId>
         <artifactId>common</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>common-io</artifactId>
     <packaging>jar</packaging>

common/common-lang/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.common</groupId>
         <artifactId>common</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>common-lang</artifactId>
     <packaging>jar</packaging>

common/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys</groupId>
         <artifactId>twelvemonkeys</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <groupId>com.twelvemonkeys.common</groupId>
     <artifactId>common</artifactId>

contrib/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys</groupId>
         <artifactId>twelvemonkeys</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <groupId>com.twelvemonkeys.contrib</groupId>
     <artifactId>contrib</artifactId>

imageio/imageio-batik/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-batik</artifactId>
     <name>TwelveMonkeys :: ImageIO :: Batik Plugin</name>

imageio/imageio-bmp/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-bmp</artifactId>
     <name>TwelveMonkeys :: ImageIO :: BMP plugin</name>

imageio/imageio-clippath/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-clippath</artifactId>
     <name>TwelveMonkeys :: ImageIO :: Photoshop Path Support</name>

imageio/imageio-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-core</artifactId>
     <name>TwelveMonkeys :: ImageIO :: Core</name>

imageio/imageio-hdr/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-hdr</artifactId>
     <name>TwelveMonkeys :: ImageIO :: HDR plugin</name>

imageio/imageio-icns/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-icns</artifactId>
     <name>TwelveMonkeys :: ImageIO :: ICNS plugin</name>

imageio/imageio-iff/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-iff</artifactId>
     <name>TwelveMonkeys :: ImageIO :: IFF plugin</name>

imageio/imageio-jpeg/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-jpeg</artifactId>
     <name>TwelveMonkeys :: ImageIO :: JPEG plugin</name>

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/Application.java

@@ -29,7 +29,6 @@
 package com.twelvemonkeys.imageio.plugins.jpeg;
 
 import com.twelvemonkeys.imageio.metadata.jpeg.JPEG;
-import com.twelvemonkeys.lang.Validate;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInput;
@@ -48,10 +47,10 @@ class Application extends Segment {
     final String identifier;
     final byte[] data;
 
-    Application(int marker, final String identifier, final byte[] data) {
+    Application(final int marker, final String identifier, final byte[] data) {
         super(marker);
 
-        this.identifier = Validate.notEmpty(identifier, "identifier");
+        this.identifier = identifier; // NOTE: Some JPEGs contain APP segments without NULL-terminated identifier
         this.data = data;
     }
 

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/HuffmanTable.java

@@ -33,41 +33,27 @@ package com.twelvemonkeys.imageio.plugins.jpeg;
 import com.twelvemonkeys.imageio.metadata.jpeg.JPEG;
 
 import javax.imageio.IIOException;
-import javax.imageio.stream.ImageInputStream;
 import java.io.DataInput;
 import java.io.IOException;
 
 final class HuffmanTable extends Segment {
 
-    final int l[][][] = new int[4][2][16];
-    final int th[] = new int[4]; // 1: this table is presented
+    private final int l[][][] = new int[4][2][16];
+    private final int th[] = new int[4]; // 1: this table is present
     final int v[][][][] = new int[4][2][16][200]; // tables
-    final int[][] tc = new int[4][2]; // 1: this table is presented
+    final int[][] tc = new int[4][2]; // 1: this table is present
 
-    public static final int MSB = 0x80000000;
+    static final int MSB = 0x80000000;
 
     private HuffmanTable() {
         super(JPEG.DHT);
+   }
 
-        tc[0][0] = 0;
-        tc[1][0] = 0;
-        tc[2][0] = 0;
-        tc[3][0] = 0;
-        tc[0][1] = 0;
-        tc[1][1] = 0;
-        tc[2][1] = 0;
-        tc[3][1] = 0;
-        th[0] = 0;
-        th[1] = 0;
-        th[2] = 0;
-        th[3] = 0;
-    }
-
-    protected void buildHuffTables(final int[][][] HuffTab) throws IOException {
-        for (int i = 0; i < 4; i++) {
-            for (int j = 0; j < 2; j++) {
-                if (tc[i][j] != 0) {
-                    buildHuffTable(HuffTab[i][j], l[i][j], v[i][j]);
+    void buildHuffTables(final int[][][] HuffTab) throws IOException {
+        for (int t = 0; t < 4; t++) {
+            for (int c = 0; c < 2; c++) {
+                if (tc[t][c] != 0) {
+                    buildHuffTable(HuffTab[t][c], l[t][c], v[t][c]);
                 }
             }
         }
@@ -81,10 +67,8 @@ final class HuffmanTable extends Segment {
     //	Effect:
     //	    build up HuffTab[t][c] using L and V.
     private void buildHuffTable(final int tab[], final int L[], final int V[][]) throws IOException {
-        int currentTable, temp;
-        int k;
-        temp = 256;
-        k = 0;
+        int temp = 256;
+        int k = 0;
 
         for (int i = 0; i < 8; i++) { // i+1 is Code length
             for (int j = 0; j < L[i]; j++) {
@@ -99,7 +83,7 @@ final class HuffmanTable extends Segment {
             tab[k] = i | MSB;
         }
 
-        currentTable = 1;
+        int currentTable = 1;
         k = 0;
 
         for (int i = 8; i < 16; i++) { // i+1 is Code length
@@ -122,8 +106,27 @@ final class HuffmanTable extends Segment {
 
     @Override
     public String toString() {
-        // TODO: Id and class for tables
-        return "DHT[]";
+        StringBuilder builder = new StringBuilder("DHT[");
+
+        for (int t = 0; t < tc.length; t++) {
+            for (int c = 0; c < tc[t].length; c++) {
+                if (tc[t][c] != 0) {
+                    if (builder.length() > 4) {
+                        builder.append(", ");
+                    }
+
+                    builder.append("id: ");
+                    builder.append(t);
+
+                    builder.append(", class: ");
+                    builder.append(c);
+                }
+            }
+        }
+
+        builder.append(']');
+
+        return builder.toString();
     }
 
     public static Segment read(final DataInput data, final int length) throws IOException {

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/JPEGImageReader.java

@@ -109,37 +109,7 @@ public class JPEGImageReader extends ImageReaderBase {
     static final int ALL_APP_MARKERS = -1;
 
     /** Segment identifiers for the JPEG segments we care about reading. */
-    private static final Map<Integer, List<String>> SEGMENT_IDENTIFIERS = JPEGSegmentUtil.ALL_SEGMENTS; //createSegmentIds();
-
-    private static Map<Integer, List<String>> createSegmentIds() {
-        Map<Integer, List<String>> map = new LinkedHashMap<>();
-
-        // Need all APP markers to be able to re-generate proper metadata later
-        for (int appMarker = JPEG.APP0; appMarker <= JPEG.APP15; appMarker++) {
-            map.put(appMarker, JPEGSegmentUtil.ALL_IDS);
-        }
-
-        // SOFn markers
-        map.put(JPEG.SOF0, null);
-        map.put(JPEG.SOF1, null);
-        map.put(JPEG.SOF2, null);
-        map.put(JPEG.SOF3, null);
-        map.put(JPEG.SOF5, null);
-        map.put(JPEG.SOF6, null);
-        map.put(JPEG.SOF7, null);
-        map.put(JPEG.SOF9, null);
-        map.put(JPEG.SOF10, null);
-        map.put(JPEG.SOF11, null);
-        map.put(JPEG.SOF13, null);
-        map.put(JPEG.SOF14, null);
-        map.put(JPEG.SOF15, null);
-
-        map.put(JPEG.DQT, null);
-        map.put(JPEG.DHT, null);
-        map.put(JPEG.SOS, null);
-
-        return Collections.unmodifiableMap(map);
-    }
+    private static final Map<Integer, List<String>> SEGMENT_IDENTIFIERS = JPEGSegmentUtil.ALL_SEGMENTS;
 
     /** Our JPEG reading delegate */
     private final ImageReader delegate;
@@ -385,7 +355,7 @@ public class JPEGImageReader extends ImageReaderBase {
             // TODO: What about stream position?
             // TODO: Param handling: Source region, offset, subsampling, destination, destination type, etc....
             // Read image as lossless
-            return new JPEGLosslessDecoderWrapper().readImage(segments, imageInput);
+            return new JPEGLosslessDecoderWrapper(this).readImage(segments, imageInput);
         }
         
         // We need to apply ICC profile unless the profile is sRGB/default gray (whatever that is)
@@ -723,7 +693,7 @@ public class JPEGImageReader extends ImageReaderBase {
                 catch (IOException e) {
                     // TODO: Handle bad segments better, for now, just ignore any bad APP markers
                     if (segment.marker() >= JPEG.APP0 && JPEG.APP15 >= segment.marker()) {
-                        processWarningOccurred("Bogus " +segment.identifier() + " segment, ignoring");
+                        processWarningOccurred("Bogus APP" + (segment.marker() & 0x0f) + "/" + segment.identifier() + " segment, ignoring");
                         continue;
                     }
 
@@ -944,7 +914,7 @@ public class JPEGImageReader extends ImageReaderBase {
         if (isLossless()) {
             // TODO: What about stream position?
             // TODO: Param handling: Reading as raster should support source region, subsampling etc.
-            return new JPEGLosslessDecoderWrapper().readRaster(segments, imageInput);
+            return new JPEGLosslessDecoderWrapper(this).readRaster(segments, imageInput);
         }
 
         return delegate.readRaster(imageIndex, param);

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/JPEGLosslessDecoder.java

@@ -36,14 +36,16 @@ import com.twelvemonkeys.lang.Validate;
 import javax.imageio.IIOException;
 import javax.imageio.stream.ImageInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 final class JPEGLosslessDecoder {
 
     private final ImageInputStream input;
+    private final JPEGImageReader listenerDelegate;
 
     private final Frame frame;
-    private final HuffmanTable huffTable;
+    private final List<HuffmanTable> huffTables;
     private final QuantizationTable quantTable;
     private Scan scan;
 
@@ -103,7 +105,7 @@ final class JPEGLosslessDecoder {
         return yDim;
     }
 
-    JPEGLosslessDecoder(final List<Segment> segments, final ImageInputStream data) {
+    JPEGLosslessDecoder(final List<Segment> segments, final ImageInputStream data, final JPEGImageReader listenerDelegate) {
         Validate.notNull(segments);
 
         frame = get(segments, Frame.class);
@@ -111,11 +113,25 @@ final class JPEGLosslessDecoder {
 
         QuantizationTable qt = get(segments, QuantizationTable.class);
         quantTable = qt != null ? qt : new QuantizationTable(); // For lossless there are no DQTs
-        huffTable = get(segments, HuffmanTable.class); // For non-lossless there can be multiple of DHTs
+        huffTables = getAll(segments, HuffmanTable.class); // For lossless there's usually only one, and only DC tables
+
         RestartInterval dri = get(segments, RestartInterval.class);
         restartInterval = dri != null ? dri.interval : 0;
 
         input = data;
+        this.listenerDelegate = listenerDelegate;
+    }
+
+    private <T> List<T> getAll(final List<Segment> segments, final Class<T> type) {
+        ArrayList<T> list = new ArrayList<>();
+
+        for (Segment segment : segments) {
+            if (type.isInstance(segment)) {
+                list.add(type.cast(segment));
+            }
+        }
+
+        return list;
     }
 
     private <T> T get(final List<Segment> segments, final Class<T> type) {
@@ -138,10 +154,13 @@ final class JPEGLosslessDecoder {
         current = input.readUnsignedShort();
 
         if (current != JPEG.SOI) { // SOI
-            throw new IIOException("Not a JPEG file");
+            throw new IIOException("Not a JPEG file, does not start with 0xFFD8");
+        }
+
+        for (HuffmanTable huffTable : huffTables) {
+            huffTable.buildHuffTables(HuffTab);
         }
 
-        huffTable.buildHuffTables(HuffTab);
         quantTable.enhanceTables(TABLE);
 
         current = input.readUnsignedShort();
@@ -175,8 +194,23 @@ final class JPEGLosslessDecoder {
                 Frame.Component component = getComponentSpec(components, scanComps[i].scanCompSel);
                 qTab[i] = quantTables[component.qtSel];
                 nBlock[i] = component.vSub * component.hSub;
-                dcTab[i] = HuffTab[scanComps[i].dcTabSel][0];
-                acTab[i] = HuffTab[scanComps[i].acTabSel][1];
+
+                int dcTabSel = scanComps[i].dcTabSel;
+                int acTabSel = scanComps[i].acTabSel;
+
+                // NOTE: If we don't find any DC tables for lossless operation, this file isn't any good.
+                // However, we have seen files with AC tables only, we'll treat these as if the AC was DC
+                if (useACForDC(dcTabSel)) {
+                    processWarningOccured("Lossless JPEG with no DC tables encountered. Assuming only tables present to be DC tables.");
+
+                    dcTab[i] = HuffTab[dcTabSel][1];
+                    acTab[i] = HuffTab[acTabSel][0];
+                }
+                else {
+                    // All good
+                    dcTab[i] = HuffTab[dcTabSel][0];
+                    acTab[i] = HuffTab[acTabSel][1];
+                }
             }
 
             xDim = frame.samplesPerLine;
@@ -202,10 +236,8 @@ final class JPEGLosslessDecoder {
             scanNum++;
 
             while (true) { // Decode one scan
-                final int temp[] = new int[1]; // to store remainder bits
-                final int index[] = new int[1];
-                temp[0] = 0;
-                index[0] = 0;
+                int temp[] = new int[1]; // to store remainder bits
+                int index[] = new int[1];
 
                 for (int i = 0; i < 10; i++) {
                     pred[i] = (1 << (precision - 1));
@@ -242,10 +274,7 @@ final class JPEGLosslessDecoder {
                     }
                 }
 
-                if ((current >= RESTART_MARKER_BEGIN) && (current <= RESTART_MARKER_END)) {
-                    //empty
-                }
-                else {
+                if ((current < RESTART_MARKER_BEGIN) || (current > RESTART_MARKER_END)) {
                     break; //current=MARKER
                 }
             }
@@ -259,6 +288,34 @@ final class JPEGLosslessDecoder {
         return outputRef;
     }
 
+    private void processWarningOccured(String warning) {
+        listenerDelegate.processWarningOccurred(warning);
+    }
+
+    private boolean useACForDC(final int dcTabSel) {
+        if (isLossless()) {
+            for (HuffmanTable huffTable : huffTables) {
+                if (huffTable.tc[dcTabSel][0] == 0 && huffTable.tc[dcTabSel][1] != 0) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    private boolean isLossless() {
+        switch (frame.marker) {
+            case JPEG.SOF3:
+            case JPEG.SOF7:
+            case JPEG.SOF11:
+            case JPEG.SOF15:
+                return true;
+            default:
+                return false;
+        }
+    }
+
     private Frame.Component getComponentSpec(Frame.Component[] components, int sel) {
         for (Frame.Component component : components) {
             if (component.id == sel) {
@@ -320,15 +377,15 @@ final class JPEGLosslessDecoder {
         }
 
         for (int i = 0; i < nBlock[0]; i++) {
-            final int value = getHuffmanValue(dcTab[0], temp, index);
+            int value = getHuffmanValue(dcTab[0], temp, index);
 
             if (value >= 0xFF00) {
                 return value;
             }
 
-            final int n = getn(prev, value, temp, index);
+            int n = getn(prev, value, temp, index);
 
-            final int nRestart = (n >> 8);
+            int nRestart = (n >> 8);
             if ((nRestart >= RESTART_MARKER_BEGIN) && (nRestart <= RESTART_MARKER_END)) {
                 return nRestart;
             }

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/JPEGLosslessDecoderWrapper.java

@@ -29,6 +29,9 @@
 
 package com.twelvemonkeys.imageio.plugins.jpeg;
 
+import com.twelvemonkeys.imageio.stream.BufferedImageInputStream;
+
+import javax.imageio.IIOException;
 import javax.imageio.stream.ImageInputStream;
 import java.awt.image.BufferedImage;
 import java.awt.image.DataBufferByte;
@@ -54,6 +57,12 @@ import java.util.List;
  */
 final class JPEGLosslessDecoderWrapper {
 
+    private final JPEGImageReader listenerDelegate;
+
+    JPEGLosslessDecoderWrapper(final JPEGImageReader listenerDelegate) {
+        this.listenerDelegate = listenerDelegate;
+    }
+
     /**
      * Decodes a JPEG Lossless stream to a {@code BufferedImage}.
      * Currently the following conversions are supported:
@@ -61,14 +70,19 @@ final class JPEGLosslessDecoderWrapper {
      * -  8Bit, Grayscale -> BufferedImage.TYPE_BYTE_GRAY
      * - 16Bit, Grayscale -> BufferedImage.TYPE_USHORT_GRAY
      *
-     * @param segments
+     * @param segments segments
      * @param input input stream which contains a jpeg lossless data
      * @return if successfully a BufferedImage is returned
      * @throws IOException is thrown if the decoder failed or a conversion is not supported
      */
     BufferedImage readImage(final List<Segment> segments, final ImageInputStream input) throws IOException {
-        JPEGLosslessDecoder decoder = new JPEGLosslessDecoder(segments, input);
+        JPEGLosslessDecoder decoder = new JPEGLosslessDecoder(segments, createBufferedInput(input), listenerDelegate);
 
+        // TODO: Allow 10/12/14 bit (using a ComponentColorModel with correct bits, as in TIFF)
+        // TODO: Rewrite this to pass a pre-allocated buffer of correct type (byte/short)/correct bands
+        // TODO: Progress callbacks
+        // TODO: Warning callbacks
+        // Callback can then do subsampling etc.
         int[][] decoded = decoder.decode();
         int width = decoder.getDimX();
         int height = decoder.getDimY();
@@ -80,23 +94,21 @@ final class JPEGLosslessDecoderWrapper {
                     return to8Bit1ComponentGrayScale(decoded, width, height);
                 case 16:
                     return to16Bit1ComponentGrayScale(decoded, width, height);
-                default:
-                    throw new IOException("JPEG Lossless with " + decoder.getPrecision() + " bit precision and 1 component cannot be decoded");
             }
         }
-
         // 3 components, assumed to be RGB
-        if (decoder.getNumComponents() == 3) {
+        else if (decoder.getNumComponents() == 3) {
             switch (decoder.getPrecision()) {
                 case 8:
                     return to24Bit3ComponentRGB(decoded, width, height);
-
-                default:
-                    throw new IOException("JPEG Lossless with " + decoder.getPrecision() + " bit precision and 3 components cannot be decoded");
             }
         }
 
-        throw new IOException("JPEG Lossless with " + decoder.getPrecision() + " bit precision and " + decoder.getNumComponents() + " component(s) cannot be decoded");
+        throw new IIOException("JPEG Lossless with " + decoder.getPrecision() + " bit precision and " + decoder.getNumComponents() + " component(s) not supported");
+    }
+
+    private ImageInputStream createBufferedInput(final ImageInputStream input) throws IOException {
+        return input instanceof BufferedImageInputStream ? input : new BufferedImageInputStream(input);
     }
 
     Raster readRaster(final List<Segment> segments, final ImageInputStream input) throws IOException {

imageio/imageio-jpeg/src/main/java/com/twelvemonkeys/imageio/plugins/jpeg/JPEGSegmentImageInputStream.java

@@ -361,7 +361,6 @@ final class JPEGSegmentImageInputStream extends ImageInputStreamImpl {
             return stream.read(b, off, len);
         }
 
-
         @Override
         public String toString() {
             return String.format("0x%04x[%d-%d]", marker, realStart, realEnd());
@@ -409,7 +408,7 @@ final class JPEGSegmentImageInputStream extends ImageInputStreamImpl {
             byte[] replacementData = new byte[length];
 
             int numQTs = length / 128;
-            int newSegmentLength = 2 + 1 + 64 * numQTs;
+            int newSegmentLength = 2 + (1 + 64) * numQTs; // Len + (qtInfo + qtSize) * numQTs
 
             replacementData[0] = (byte) ((JPEG.DQT >> 8) & 0xff);
             replacementData[1] = (byte) (JPEG.DQT & 0xff);

imageio/imageio-jpeg/src/test/java/com/twelvemonkeys/imageio/plugins/jpeg/JPEGImageReaderTest.java

@@ -94,10 +94,12 @@ public class JPEGImageReaderTest extends ImageReaderAbstractTest<JPEGImageReader
                 new TestData(getClassLoaderResource("/jpeg/jfif-padded-segments.jpg"), new Dimension(20, 45)),
                 new TestData(getClassLoaderResource("/jpeg/0x00-to-0xFF-between-segments.jpg"), new Dimension(16, 16)),
                 new TestData(getClassLoaderResource("/jpeg/jfif-bogus-empty-jfif-segment.jpg"), new Dimension(942, 714)),
+                new TestData(getClassLoaderResource("/jpeg/app-marker-missing-null-term.jpg"), new Dimension(200, 150)),
                 new TestData(getClassLoaderResource("/jpeg/jfif-16bit-dqt.jpg"), new Dimension(204, 131)),
                 new TestData(getClassLoaderResource("/jpeg-lossless/8_ls.jpg"), new Dimension(800, 535)),  // Lossless gray, 8 bit
                 new TestData(getClassLoaderResource("/jpeg-lossless/16_ls.jpg"), new Dimension(800, 535)),  // Lossless gray, 16 bit
                 new TestData(getClassLoaderResource("/jpeg-lossless/24_ls.jpg"), new Dimension(800, 535)), // Lossless RGB, 8 bit per component (24 bit)
+                new TestData(getClassLoaderResource("/jpeg-lossless/f-18.jpg"), new Dimension(320, 240)), // Lossless RGB, 3 DHTs
                 new TestData(getClassLoaderResource("/jpeg-lossless/testimg_rgb.jpg"), new Dimension(227, 149)), // Lossless RGB, 8 bit per component (24 bit)
                 new TestData(getClassLoaderResource("/jpeg-lossless/testimg_gray.jpg"), new Dimension(512, 512)) // Lossless gray, 16 bit
         );

imageio/imageio-metadata/pom.xml

@@ -3,7 +3,7 @@
     <parent>
 	<groupId>com.twelvemonkeys.imageio</groupId>        
 	<artifactId>imageio</artifactId>        
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>imageio-metadata</artifactId>

imageio/imageio-metadata/src/main/java/com/twelvemonkeys/imageio/metadata/xmp/XMPReader.java

@@ -28,25 +28,34 @@
 
 package com.twelvemonkeys.imageio.metadata.xmp;
 
-import com.twelvemonkeys.imageio.metadata.Directory;
-import com.twelvemonkeys.imageio.metadata.Entry;
-import com.twelvemonkeys.imageio.metadata.MetadataReader;
-import com.twelvemonkeys.imageio.util.IIOUtil;
-import com.twelvemonkeys.lang.Validate;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.imageio.IIOException;
+import javax.imageio.stream.ImageInputStream;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
 import org.w3c.dom.Document;
 import org.w3c.dom.NamedNodeMap;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
 
-import javax.imageio.IIOException;
-import javax.imageio.stream.ImageInputStream;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.IOException;
-import java.util.*;
+import com.twelvemonkeys.imageio.metadata.Directory;
+import com.twelvemonkeys.imageio.metadata.Entry;
+import com.twelvemonkeys.imageio.metadata.MetadataReader;
+import com.twelvemonkeys.imageio.util.IIOUtil;
+import com.twelvemonkeys.lang.Validate;
 
 /**
  * XMPReader
@@ -64,20 +73,17 @@ public final class XMPReader extends MetadataReader {
     public Directory read(final ImageInputStream input) throws IOException {
         Validate.notNull(input, "input");
 
-        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-        factory.setNamespaceAware(true);
-
         try {
+            DocumentBuilderFactory factory = createDocumentBuilderFactory();
+
             // TODO: Consider parsing using SAX?
             // TODO: Determine encoding and parse using a Reader...
             // TODO: Refactor scanner to return inputstream?
             // TODO: Be smarter about ASCII-NULL termination/padding (the SAXParser aka Xerces DOMParser doesn't like it)...
             DocumentBuilder builder = factory.newDocumentBuilder();
+            builder.setErrorHandler(new DefaultHandler());
             Document document = builder.parse(new InputSource(IIOUtil.createStreamAdapter(input)));
 
-//            XMLSerializer serializer = new XMLSerializer(System.err, System.getProperty("file.encoding"));
-//            serializer.serialize(document);
-
             String toolkit = getToolkit(document);
             Node rdfRoot = document.getElementsByTagNameNS(XMP.NS_RDF, "RDF").item(0);
             NodeList descriptions = document.getElementsByTagNameNS(XMP.NS_RDF, "Description");
@@ -88,10 +94,33 @@ public final class XMPReader extends MetadataReader {
             throw new IIOException(e.getMessage(), e);
         }
         catch (ParserConfigurationException e) {
-            throw new RuntimeException(e); // TODO: Or IOException?
+            throw new RuntimeException(e);
         }
     }
 
+    private DocumentBuilderFactory createDocumentBuilderFactory() throws ParserConfigurationException {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        factory.setNamespaceAware(true);
+
+        // Security: Disable XInclude & expanding entity references ("bombs"), not needed for XMP
+        factory.setXIncludeAware(false);
+        factory.setExpandEntityReferences(false);
+
+        // Security: Enable "secure processing", to prevent DoS attacks
+        factory.setAttribute(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+
+        // Security: Remove possibility to access external DTDs or Schema, not needed for XMP
+        factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+
+        // Security: Disable loading of external DTD and entities, not needed for XMP
+        factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+
+        return factory;
+    }
+
     private String getToolkit(Document document) {
         NodeList xmpmeta = document.getElementsByTagNameNS(XMP.NS_X, "xmpmeta");
 
@@ -105,7 +134,7 @@ public final class XMPReader extends MetadataReader {
     }
 
     private XMPDirectory parseDirectories(final Node pParentNode, NodeList pNodes, String toolkit) {
-        Map<String, List<Entry>> subdirs = new LinkedHashMap<String, List<Entry>>();
+        Map<String, List<Entry>> subdirs = new LinkedHashMap<>();
 
         for (Node desc : asIterable(pNodes)) {
             if (desc.getParentNode() != pParentNode) {
@@ -123,7 +152,7 @@ public final class XMPReader extends MetadataReader {
                 // Lookup
                 List<Entry> dir = subdirs.get(node.getNamespaceURI());
                 if (dir == null) {
-                    dir = new ArrayList<Entry>();
+                    dir = new ArrayList<>();
                     subdirs.put(node.getNamespaceURI(), dir);
                 }
 
@@ -135,12 +164,12 @@ public final class XMPReader extends MetadataReader {
                 else {
                     // TODO: This method contains loads of duplication an should be cleaned up...
                     // Support attribute short-hand syntax
-                    Map<String, List<Entry>> subsubdirs = new LinkedHashMap<String, List<Entry>>();
+                    Map<String, List<Entry>> subsubdirs = new LinkedHashMap<>();
 
                     parseAttributesForKnownElements(subsubdirs, node);
 
                     if (!subsubdirs.isEmpty()) {
-                        List<Entry> entries = new ArrayList<Entry>();
+                        List<Entry> entries = new ArrayList<>(subsubdirs.size());
 
                         for (Map.Entry<String, List<Entry>> entry : subsubdirs.entrySet()) {
                             entries.addAll(entry.getValue());
@@ -157,7 +186,7 @@ public final class XMPReader extends MetadataReader {
             }
         }
 
-        List<Directory> entries = new ArrayList<Directory>();
+        List<Directory> entries = new ArrayList<>(subdirs.size());
 
         // TODO: Should we still allow asking for a subdirectory by item id?
         for (Map.Entry<String, List<Entry>> entry : subdirs.entrySet()) {
@@ -175,7 +204,7 @@ public final class XMPReader extends MetadataReader {
 
     private RDFDescription parseAsResource(Node node) {
         // See: http://www.w3.org/TR/REC-rdf-syntax/#section-Syntax-parsetype-resource
-        List<Entry> entries = new ArrayList<Entry>();
+        List<Entry> entries = new ArrayList<>();
 
         for (Node child : asIterable(node.getChildNodes())) {
             if (child.getNodeType() != Node.ELEMENT_NODE) {
@@ -200,7 +229,7 @@ public final class XMPReader extends MetadataReader {
             List<Entry> dir = subdirs.get(attr.getNamespaceURI());
 
             if (dir == null) {
-                dir = new ArrayList<Entry>();
+                dir = new ArrayList<>();
                 subdirs.put(attr.getNamespaceURI(), dir);
             }
 
@@ -212,7 +241,7 @@ public final class XMPReader extends MetadataReader {
         for (Node child : asIterable(node.getChildNodes())) {
             if (XMP.NS_RDF.equals(child.getNamespaceURI()) && "Alt".equals(child.getLocalName())) {
                 // Support for <rdf:Alt><rdf:li> -> return a Map<String, Object> keyed on xml:lang
-                Map<String, Object> alternatives = new LinkedHashMap<String, Object>();
+                Map<String, Object> alternatives = new LinkedHashMap<>();
                 for (Node alternative : asIterable(child.getChildNodes())) {
                     if (XMP.NS_RDF.equals(alternative.getNamespaceURI()) && "li".equals(alternative.getLocalName())) {
                         NamedNodeMap attributes = alternative.getAttributes();
@@ -226,7 +255,7 @@ public final class XMPReader extends MetadataReader {
             else if (XMP.NS_RDF.equals(child.getNamespaceURI()) && ("Seq".equals(child.getLocalName()) || "Bag".equals(child.getLocalName()))) {
                 // Support for <rdf:Seq><rdf:li> -> return array
                 // Support for <rdf:Bag><rdf:li> -> return array/unordered collection (how can a serialized collection not have order?)
-                List<Object> seq = new ArrayList<Object>();
+                List<Object> seq = new ArrayList<>();
 
                 for (Node sequence : asIterable(child.getChildNodes())) {
                     if (XMP.NS_RDF.equals(sequence.getNamespaceURI()) && "li".equals(sequence.getLocalName())) {

imageio/imageio-metadata/src/test/java/com/twelvemonkeys/imageio/metadata/xmp/XMPReaderTest.java

@@ -28,23 +28,32 @@
 
 package com.twelvemonkeys.imageio.metadata.xmp;
 
-import com.twelvemonkeys.imageio.metadata.CompoundDirectory;
-import com.twelvemonkeys.imageio.metadata.Directory;
-import com.twelvemonkeys.imageio.metadata.Entry;
-import com.twelvemonkeys.imageio.metadata.MetadataReaderAbstractTest;
-import org.junit.Test;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
-import javax.imageio.ImageIO;
-import javax.imageio.stream.ImageInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.junit.Assert.*;
+import javax.imageio.ImageIO;
+import javax.imageio.stream.ImageInputStream;
+
+import org.junit.Test;
+
+import com.twelvemonkeys.imageio.metadata.CompoundDirectory;
+import com.twelvemonkeys.imageio.metadata.Directory;
+import com.twelvemonkeys.imageio.metadata.Entry;
+import com.twelvemonkeys.imageio.metadata.MetadataReaderAbstractTest;
 
 /**
  * XMPReaderTest
@@ -375,6 +384,7 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
         // photoshop|http://ns.adobe.com/photoshop/1.0/
         Directory photoshop = getDirectoryByNS(compound, XMP.NS_PHOTOSHOP);
 
+        assertNotNull(photoshop);
         assertEquals(3, photoshop.size());
         assertThat(photoshop.getEntryById("http://ns.adobe.com/photoshop/1.0/ColorMode"), hasValue("1"));
         assertThat(photoshop.getEntryById("http://ns.adobe.com/photoshop/1.0/ICCProfile"), hasValue("Dot Gain 20%"));
@@ -390,6 +400,8 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
 
         // xapMM|http://ns.adobe.com/xap/1.0/mm/
         Directory mm = getDirectoryByNS(compound, XMP.NS_XAP_MM);
+
+        assertNotNull(mm);
         assertEquals(3, mm.size());
         assertThat(directory.getEntryById("http://ns.adobe.com/xap/1.0/mm/DocumentID"), hasValue("uuid:6DCA50CC7D53DD119F20F5A7EA4C9BEC"));
         assertThat(directory.getEntryById("http://ns.adobe.com/xap/1.0/mm/InstanceID"), hasValue("uuid:6ECA50CC7D53DD119F20F5A7EA4C9BEC"));
@@ -414,6 +426,8 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
 
         // dc|http://purl.org/dc/elements/1.1/
         Directory dc = getDirectoryByNS(compound, XMP.NS_DC);
+
+        assertNotNull(dc);
         assertEquals(1, dc.size());
 
         assertThat(dc.getEntryById("http://purl.org/dc/elements/1.1/format"), hasValue("image/jpeg"));
@@ -428,6 +442,8 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
 
         // tiff|http://ns.adobe.com/tiff/1.0/
         Directory tiff = getDirectoryByNS(compound, XMP.NS_TIFF);
+
+        assertNotNull(tiff);
         assertEquals(5, tiff.size());
         assertThat(tiff.getEntryById("http://ns.adobe.com/tiff/1.0/Orientation"), hasValue("1"));
         assertThat(tiff.getEntryById("http://ns.adobe.com/tiff/1.0/XResolution"), hasValue("720000/10000"));
@@ -445,6 +461,8 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
 
         // xap|http://ns.adobe.com/xap/1.0/
         Directory xap = getDirectoryByNS(compound, XMP.NS_XAP);
+
+        assertNotNull(xap);
         assertEquals(4, xap.size());
         assertThat(xap.getEntryById("http://ns.adobe.com/xap/1.0/ModifyDate"), hasValue("2008-07-16T14:44:49-07:00"));
         assertThat(xap.getEntryById("http://ns.adobe.com/xap/1.0/CreatorTool"), hasValue("Adobe Photoshop CS3 Windows"));
@@ -461,10 +479,77 @@ public class XMPReaderTest extends MetadataReaderAbstractTest {
 
         // exif|http://ns.adobe.com/exif/1.0/
         Directory exif = getDirectoryByNS(compound, XMP.NS_EXIF);
+
+        assertNotNull(exif);
         assertEquals(4, exif.size());
         assertThat(exif.getEntryById("http://ns.adobe.com/exif/1.0/ColorSpace"), hasValue("-1")); // SIC. Same as unsigned short 65535, meaning "uncalibrated"?
         assertThat(exif.getEntryById("http://ns.adobe.com/exif/1.0/PixelXDimension"), hasValue("426"));
         assertThat(exif.getEntryById("http://ns.adobe.com/exif/1.0/PixelYDimension"), hasValue("550"));
         assertThat(exif.getEntryById("http://ns.adobe.com/exif/1.0/NativeDigest"), hasValue("36864,40960,40961,37121,37122,40962,40963,37510,40964,36867,36868,33434,33437,34850,34852,34855,34856,37377,37378,37379,37380,37381,37382,37383,37384,37385,37386,37396,41483,41484,41486,41487,41488,41492,41493,41495,41728,41729,41730,41985,41986,41987,41988,41989,41990,41991,41992,41993,41994,41995,41996,42016,0,2,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,23,24,25,26,27,28,30;A7F21D25E2C562F152B2C4ECC9E534DA"));
     }
+
+    @Test(timeout = 1500L)
+    public void testNoExternalRequest() throws Exception {
+        // TODO: Use dynamic port?
+        try (HTTPServer server = new HTTPServer(7777)) {
+            try {
+                createReader().read(getResourceAsIIS("/xmp/xmp-jpeg-xxe.xml"));
+            } catch (IOException ioe) {
+                if (ioe.getMessage().contains("501")) {
+                    throw new AssertionError("Reading should not cause external requests", ioe);
+                }
+
+                // Any other exception is a bug (but might happen if the parser does not support certain features)
+                throw ioe;
+            }
+        }
+    }
+
+    private static class HTTPServer implements AutoCloseable {
+        private final ServerSocket server;
+        private final Thread thread;
+
+        HTTPServer(int port) throws IOException {
+            server = new ServerSocket(port, 1);
+            thread = new Thread(new Runnable() {
+                @Override public void run() {
+                    serve();
+                }
+            });
+            thread.start();
+        }
+
+        private void serve() {
+            try {
+                Socket client = server.accept();
+
+                // Get the input stream, don't care about the request
+                try (InputStream inputStream = client.getInputStream()) {
+                    while (inputStream.available() > 0) {
+                        if (inputStream.read() == -1) {
+                            break;
+                        }
+                    }
+
+                    // Answer with 501, this will cause the client to throw IOException
+                    try (OutputStream outputStream = client.getOutputStream()) {
+                        outputStream.write("HTTP/1.0 501 Not Implemented\r\n\r\n".getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+            }
+            catch (IOException e) {
+                if (server.isClosed() && e instanceof SocketException) {
+                    // Socket closed due to server close, all good
+                    return;
+                }
+
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override public void close() throws Exception {
+            server.close();
+            thread.join(); // It's advised against throwing InterruptedException here, but this is not production code...
+        }
+    }
 }

imageio/imageio-metadata/src/test/resources/xmp/xmp-jpeg-xxe.xml

@@ -0,0 +1,35 @@
+<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?><!DOCTYPE root [<!ENTITY % ext SYSTEM 'http://localhost:7777/xxx'> %ext;]>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='Image::ExifTool 10.16'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'>
+
+ <rdf:Description rdf:about=''
+  xmlns:xmpMM='http://ns.adobe.com/xap/1.0/mm/'>
+  <xmpMM:InstanceID>xmp.iid:7EDC21BF-371B-4189-90AF-C83A54A6A190</xmpMM:InstanceID>
+ </rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+                                                                                                    
+<?xpacket end='w'?>

imageio/imageio-pcx/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-pcx</artifactId>
     <name>TwelveMonkeys :: ImageIO :: PCX plugin</name>

imageio/imageio-pdf/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-pdf</artifactId>
     <name>TwelveMonkeys :: ImageIO :: PDF plugin</name>

imageio/imageio-pict/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-pict</artifactId>
     <name>TwelveMonkeys :: ImageIO :: PICT plugin</name>

imageio/imageio-pnm/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-pnm</artifactId>
     <name>TwelveMonkeys :: ImageIO :: PNM plugin</name>

imageio/imageio-psd/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-psd</artifactId>
     <name>TwelveMonkeys :: ImageIO :: PSD plugin</name>

imageio/imageio-reference/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-reference</artifactId>
     <name>TwelveMonkeys :: ImageIO :: reference test cases</name>

imageio/imageio-sgi/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-sgi</artifactId>
     <name>TwelveMonkeys :: ImageIO :: SGI plugin</name>

imageio/imageio-tga/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-tga</artifactId>
     <name>TwelveMonkeys :: ImageIO :: TGA plugin</name>

imageio/imageio-thumbsdb/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-thumbsdb</artifactId>
     <name>TwelveMonkeys :: ImageIO :: Thumbs.db plugin</name>

imageio/imageio-tiff/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.twelvemonkeys.imageio</groupId>
         <artifactId>imageio</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>imageio-tiff</artifactId>
     <name>TwelveMonkeys :: ImageIO :: TIFF plugin</name>

imageio/imageio-tiff/src/main/java/com/twelvemonkeys/imageio/plugins/tiff/HorizontalDeDifferencingStream.java

@@ -38,6 +38,8 @@ import java.nio.ByteOrder;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 
+import static com.twelvemonkeys.imageio.plugins.tiff.HorizontalDifferencingStream.isValidBPS;
+
 /**
  * A decoder for data converted using "horizontal differencing predictor".
  *
@@ -67,21 +69,6 @@ final class HorizontalDeDifferencingStream extends InputStream {
         buffer.flip();
     }
 
-    private boolean isValidBPS(final int bitsPerSample) {
-        switch (bitsPerSample) {
-            case 1:
-            case 2:
-            case 4:
-            case 8:
-            case 16:
-            case 32:
-            case 64:
-                return true;
-            default:
-                return false;
-        }
-    }
-
     @SuppressWarnings("StatementWithEmptyBody")
     private boolean fetch() throws IOException {
         buffer.clear();

imageio/imageio-tiff/src/main/java/com/twelvemonkeys/imageio/plugins/tiff/HorizontalDifferencingStream.java

@@ -66,7 +66,7 @@ final class HorizontalDifferencingStream extends OutputStream {
         buffer = ByteBuffer.allocate((columns * samplesPerPixel * bitsPerSample + 7) / 8).order(byteOrder);
     }
 
-    private boolean isValidBPS(final int bitsPerSample) {
+    static boolean isValidBPS(final int bitsPerSample) {
         switch (bitsPerSample) {
             case 1:
             case 2:

imageio/imageio-tiff/src/main/java/com/twelvemonkeys/imageio/plugins/tiff/TIFFCustom.java

@@ -52,6 +52,10 @@ interface TIFFCustom {
     int COMPRESSION_JPEG2000 = 34712;
     // TODO: Aperio SVS JPEG2000: 33003 (YCbCr) and 33005 (RGB), see http://openslide.org/formats/aperio/
 
+    // PIXTIFF aka DELL PixTools, see https://community.emc.com/message/515755#515755
+    /** PIXTIFF proprietary ZIP compression, identical to Deflate/ZLib. */
+    int COMPRESSION_PIXTIFF_ZIP = 50013;
+
     int PHOTOMETRIC_LOGL = 32844;
     int PHOTOMETRIC_LOGLUV = 32845;
 

imageio/imageio-tiff/src/main/java/com/twelvemonkeys/imageio/plugins/tiff/TIFFImageReader.java

@@ -815,20 +815,20 @@ public class TIFFImageReader extends ImageReaderBase {
         Boolean needsCSConversion = null;
 
         switch (compression) {
-            // TIFF Baseline
             case TIFFBaseline.COMPRESSION_NONE:
                 // No compression
-            case TIFFExtension.COMPRESSION_DEFLATE:
-                // 'PKZIP-style' Deflate
             case TIFFBaseline.COMPRESSION_PACKBITS:
                 // PackBits
             case TIFFExtension.COMPRESSION_LZW:
                 // LZW
             case TIFFExtension.COMPRESSION_ZLIB:
                 // 'Adobe-style' Deflate
+            case TIFFExtension.COMPRESSION_DEFLATE:
+                // 'PKZIP-style' Deflate
+            case TIFFCustom.COMPRESSION_PIXTIFF_ZIP:
+                // PIXTIFF proprietary 'ZIP' compression, same as Deflate
             case TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE:
                 // CCITT modified Huffman
-                // Additionally, the specification defines these values as part of the TIFF extensions:
             case TIFFExtension.COMPRESSION_CCITT_T4:
                 // CCITT Group 3 fax encoding
             case TIFFExtension.COMPRESSION_CCITT_T6:
@@ -1008,7 +1008,7 @@ public class TIFFImageReader extends ImageReaderBase {
                                 // TODO: If we have non-standard reference B/W or yCbCr coefficients,
                                 // we might still have to do extra color space conversion...
                                 if (needsCSConversion == null) {
-                                    needsCSConversion = needsCSConversion(interpretation, jpegReader.getImageMetadata(0));
+                                    needsCSConversion = needsCSConversion(interpretation, readJPEGMetadataSafe(jpegReader));
                                 }
 
                                 if (!needsCSConversion) {
@@ -1146,7 +1146,7 @@ public class TIFFImageReader extends ImageReaderBase {
                         jpegParam.setSourceSubsampling(xSub, ySub, 0, 0);
 
                         if (needsCSConversion == null) {
-                            needsCSConversion = needsCSConversion(interpretation, jpegReader.getImageMetadata(0));
+                            needsCSConversion = needsCSConversion(interpretation, readJPEGMetadataSafe(jpegReader));
                         }
 
                         if (!needsCSConversion) {
@@ -1251,7 +1251,7 @@ public class TIFFImageReader extends ImageReaderBase {
                                     Point offset = new Point(col - srcRegion.x, row - srcRegion.y);
 
                                     if (needsCSConversion == null) {
-                                        needsCSConversion = needsCSConversion(interpretation, jpegReader.getImageMetadata(0));
+                                        needsCSConversion = needsCSConversion(interpretation, readJPEGMetadataSafe(jpegReader));
                                     }
 
                                     if (!needsCSConversion) {
@@ -1319,6 +1319,17 @@ public class TIFFImageReader extends ImageReaderBase {
         return destination;
     }
 
+    private IIOMetadata readJPEGMetadataSafe(final ImageReader jpegReader) throws IOException {
+        try {
+            return jpegReader.getImageMetadata(0);
+        }
+        catch (IIOException e) {
+            processWarningOccurred("Could not read metadata metadata JPEG compressed TIFF: " + e.getMessage() + " colors may look incorrect");
+
+            return null;
+        }
+    }
+
     private boolean needsCSConversion(final int photometricInterpretation, final IIOMetadata imageMetadata) throws IOException {
         if (imageMetadata == null) {
             // Assume we're ok
@@ -1918,6 +1929,7 @@ public class TIFFImageReader extends ImageReaderBase {
             case TIFFExtension.COMPRESSION_ZLIB:
             case TIFFExtension.COMPRESSION_DEFLATE:
                 // TIFF specification, supplement 2 says ZLIB (8) and DEFLATE (32946) algorithms are identical
+            case TIFFCustom.COMPRESSION_PIXTIFF_ZIP:
                 return new InflaterInputStream(stream, new Inflater(), 1024);
             case TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE:
             case TIFFExtension.COMPRESSION_CCITT_T4:

imageio/imageio-tiff/src/main/java/com/twelvemonkeys/imageio/plugins/tiff/TIFFImageWriter.java

@@ -38,11 +38,13 @@ import com.twelvemonkeys.imageio.metadata.exif.Rational;
 import com.twelvemonkeys.imageio.metadata.exif.TIFF;
 import com.twelvemonkeys.imageio.stream.SubImageOutputStream;
 import com.twelvemonkeys.imageio.util.IIOUtil;
+import com.twelvemonkeys.imageio.util.ProgressListenerBase;
 import com.twelvemonkeys.io.enc.EncoderStream;
 import com.twelvemonkeys.io.enc.PackBitsEncoder;
 import com.twelvemonkeys.lang.Validate;
 
 import javax.imageio.*;
+import javax.imageio.event.IIOWriteWarningListener;
 import javax.imageio.metadata.IIOInvalidTreeException;
 import javax.imageio.metadata.IIOMetadata;
 import javax.imageio.metadata.IIOMetadataFormatImpl;
@@ -68,19 +70,12 @@ import java.util.zip.DeflaterOutputStream;
  * @version $Id: TIFFImageWriter.java,v 1.0 18.09.13 12:46 haraldk Exp$
  */
 public final class TIFFImageWriter extends ImageWriterBase {
-    // Short term
-    // TODO: Support more of the ImageIO metadata (ie. compression from metadata, etc)
-
     // Long term
     // TODO: Support tiling
     // TODO: Support thumbnails
-    // TODO: Support CCITT Modified Huffman compression (2)
-    // TODO: Full "Baseline TIFF" support (pending CCITT compression 2)
-    // TODO: CCITT compressions T.4 and T.6
     // TODO: Support JPEG compression of CMYK data (pending JPEGImageWriter CMYK write support)
     // ----
-    // TODO: Support storing multiple images in one stream (multi-page TIFF)
-    // TODO: Support use-case: Transcode multi-layer PSD to multi-page TIFF with metadata
+    // TODO: Support use-case: Transcode multi-layer PSD to multi-page TIFF with metadata (hard, as Photoshop don't store layers as multi-page TIFF...)
     // TODO: Support use-case: Transcode multi-page TIFF to multiple single-page TIFFs with metadata
     // TODO: Support use-case: Losslessly transcode JPEG to JPEG-in-TIFF with (EXIF) metadata (and back)
 
@@ -98,13 +93,20 @@ public final class TIFFImageWriter extends ImageWriterBase {
     // Support JPEG compression (7) - might need extra input to allow multiple images with single DQT
     // Use sensible defaults for compression based on input? None is sensible... :-)
     // Support resolution, resolution unit and software tags from ImageIO metadata
+    // Support CCITT Modified Huffman compression (2)
+    // Full "Baseline TIFF" support (pending CCITT compression 2)
+    // CCITT compressions T.4 and T.6
+    // Support storing multiple images in one stream (multi-page TIFF)
+    // Support more of the ImageIO metadata (ie. compression from metadata, etc)
 
-    public static final Rational STANDARD_DPI = new Rational(72);
+    private static final Rational STANDARD_DPI = new Rational(72);
 
     /**
      * Flag for active sequence writing
      */
-    private boolean isWritingSequence = false;
+    private boolean writingSequence = false;
+
+    private int sequenceIndex = 0;
 
     /**
      * Metadata writer for sequence writing
@@ -203,12 +205,12 @@ public final class TIFFImageWriter extends ImageWriterBase {
         EXIFWriter exifWriter = new EXIFWriter();
         exifWriter.writeTIFFHeader(imageOutput);
 
-        writePage(image, param, exifWriter, imageOutput.getStreamPosition());
+        writePage(0, image, param, exifWriter, imageOutput.getStreamPosition());
 
         imageOutput.flush();
     }
 
-    private long writePage(IIOImage image, ImageWriteParam param, EXIFWriter exifWriter, long lastIFDPointerOffset)
+    private long writePage(int imageIndex, IIOImage image, ImageWriteParam param, EXIFWriter exifWriter, long lastIFDPointerOffset)
             throws IOException {
         RenderedImage renderedImage = image.getRenderedImage();
 
@@ -262,7 +264,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
         int compression;
         if ((param == null || param.getCompressionMode() == TIFFImageWriteParam.MODE_COPY_FROM_METADATA)
                 && image.getMetadata() != null && metadata.getIFD().getEntryById(TIFF.TAG_COMPRESSION) != null) {
-            compression = (int) metadata.getIFD().getEntryById(TIFF.TAG_COMPRESSION).getValue();
+            compression = ((Number) metadata.getIFD().getEntryById(TIFF.TAG_COMPRESSION).getValue()).intValue();
         }
         else {
             compression = TIFFImageWriteParam.getCompressionType(param);
@@ -307,12 +309,13 @@ public final class TIFFImageWriter extends ImageWriterBase {
             default:
         }
 
-        // TODO: We might want to support CMYK in JPEG as well... Pending JPEG CMYK write support.
-        int photometric = compression == TIFFExtension.COMPRESSION_JPEG ? TIFFExtension.PHOTOMETRIC_YCBCR : getPhotometricInterpretation(colorModel);
+        int photometric = getPhotometricInterpretation(colorModel, compression);
         entries.put(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, new TIFFEntry(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, photometric));
 
         if (photometric == TIFFBaseline.PHOTOMETRIC_PALETTE && colorModel instanceof IndexColorModel) {
-            entries.put(TIFF.TAG_COLOR_MAP, new TIFFEntry(TIFF.TAG_COLOR_MAP, createColorMap((IndexColorModel) colorModel)));
+            // TODO: Fix consistency between sampleModel.getSampleSize() and colorModel.getPixelSize()...
+            // We should be able to support 1, 2, 4 and 8 bits per sample at least, and probably 3, 5, 6 and 7 too
+            entries.put(TIFF.TAG_COLOR_MAP, new TIFFEntry(TIFF.TAG_COLOR_MAP, createColorMap((IndexColorModel) colorModel, sampleModel.getSampleSize(0))));
             entries.put(TIFF.TAG_SAMPLES_PER_PIXEL, new TIFFEntry(TIFF.TAG_SAMPLES_PER_PIXEL, 1));
         }
         else {
@@ -419,6 +422,9 @@ public final class TIFFImageWriter extends ImageWriterBase {
             ImageWriter jpegWriter = writers.next();
             try {
                 jpegWriter.setOutput(new SubImageOutputStream(imageOutput));
+                ListenerDelegate listener = new ListenerDelegate(imageIndex);
+                jpegWriter.addIIOWriteProgressListener(listener);
+                jpegWriter.addIIOWriteWarningListener(listener);
                 jpegWriter.write(renderedImage);
             }
             finally {
@@ -427,7 +433,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
         }
         else {
             // Write image data
-            writeImageData(createCompressorStream(renderedImage, param, entries), renderedImage, numBands, bandOffsets, bitOffsets);
+            writeImageData(createCompressorStream(renderedImage, param, entries), imageIndex, renderedImage, numBands, bandOffsets, bitOffsets);
         }
 
         long stripByteCount = imageOutput.getStreamPosition() - stripOffset;
@@ -517,9 +523,12 @@ public final class TIFFImageWriter extends ImageWriterBase {
         output.length: 12600399
          */
 
+        int samplesPerPixel = (Integer) entries.get(TIFF.TAG_SAMPLES_PER_PIXEL).getValue();
+        int bitPerSample = ((short[]) entries.get(TIFF.TAG_BITS_PER_SAMPLE).getValue())[0];
+
         // Use predictor by default for LZW and ZLib/Deflate
         // TODO: Unless explicitly disabled in TIFFImageWriteParam
-        int compression = (int) entries.get(TIFF.TAG_COMPRESSION).getValue();
+        int compression = ((Number) entries.get(TIFF.TAG_COMPRESSION).getValue()).intValue();
         OutputStream stream;
 
         switch (compression) {
@@ -555,16 +564,16 @@ public final class TIFFImageWriter extends ImageWriterBase {
                 stream = IIOUtil.createStreamAdapter(imageOutput);
                 stream = new DeflaterOutputStream(stream, new Deflater(deflateSetting), 1024);
                 if (entries.containsKey(TIFF.TAG_PREDICTOR) && entries.get(TIFF.TAG_PREDICTOR).getValue().equals(TIFFExtension.PREDICTOR_HORIZONTAL_DIFFERENCING)) {
-                    stream = new HorizontalDifferencingStream(stream, image.getTileWidth(), image.getTile(0, 0).getNumBands(), image.getColorModel().getComponentSize(0), imageOutput.getByteOrder());
+                    stream = new HorizontalDifferencingStream(stream, image.getTileWidth(), samplesPerPixel, bitPerSample, imageOutput.getByteOrder());
                 }
 
                 return new DataOutputStream(stream);
 
             case TIFFExtension.COMPRESSION_LZW:
                 stream = IIOUtil.createStreamAdapter(imageOutput);
-                stream = new EncoderStream(stream, new LZWEncoder((image.getTileWidth() * image.getTileHeight() * image.getColorModel().getPixelSize() + 7) / 8));
+                stream = new EncoderStream(stream, new LZWEncoder(((image.getTileWidth() * samplesPerPixel * bitPerSample + 7) / 8) * image.getTileHeight()));
                 if (entries.containsKey(TIFF.TAG_PREDICTOR) && entries.get(TIFF.TAG_PREDICTOR).getValue().equals(TIFFExtension.PREDICTOR_HORIZONTAL_DIFFERENCING)) {
-                    stream = new HorizontalDifferencingStream(stream, image.getTileWidth(), image.getTile(0, 0).getNumBands(), image.getColorModel().getComponentSize(0), imageOutput.getByteOrder());
+                    stream = new HorizontalDifferencingStream(stream, image.getTileWidth(), samplesPerPixel, bitPerSample, imageOutput.getByteOrder());
                 }
 
                 return new DataOutputStream(stream);
@@ -575,7 +584,8 @@ public final class TIFFImageWriter extends ImageWriterBase {
                 long option = 0L;
 
                 if (compression != TIFFBaseline.COMPRESSION_CCITT_MODIFIED_HUFFMAN_RLE) {
-                    option = (long) entries.get(compression == TIFFExtension.COMPRESSION_CCITT_T4 ? TIFF.TAG_GROUP3OPTIONS : TIFF.TAG_GROUP4OPTIONS).getValue();
+                    Entry optionsEntry = entries.get(compression == TIFFExtension.COMPRESSION_CCITT_T4 ? TIFF.TAG_GROUP3OPTIONS : TIFF.TAG_GROUP4OPTIONS);
+                    option = ((Number) optionsEntry.getValue()).longValue();
                 }
 
                 Entry fillOrderEntry = entries.get(TIFF.TAG_FILL_ORDER);
@@ -589,7 +599,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
         throw new IllegalArgumentException(String.format("Unsupported TIFF compression: %d", compression));
     }
 
-    private int getPhotometricInterpretation(final ColorModel colorModel) {
+    private int getPhotometricInterpretation(final ColorModel colorModel, int compression) {
         if (colorModel.getPixelSize() == 1) {
             if (colorModel instanceof IndexColorModel) {
                 if (colorModel.getRGB(0) == 0xFFFFFFFF && colorModel.getRGB(1) == 0xFF000000) {
@@ -611,7 +621,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
             case ColorSpace.TYPE_GRAY:
                 return TIFFBaseline.PHOTOMETRIC_BLACK_IS_ZERO;
             case ColorSpace.TYPE_RGB:
-                return TIFFBaseline.PHOTOMETRIC_RGB;
+                return compression == TIFFExtension.COMPRESSION_JPEG ? TIFFExtension.PHOTOMETRIC_YCBCR : TIFFBaseline.PHOTOMETRIC_RGB;
             case ColorSpace.TYPE_CMYK:
                 return TIFFExtension.PHOTOMETRIC_SEPARATED;
         }
@@ -619,12 +629,12 @@ public final class TIFFImageWriter extends ImageWriterBase {
         throw new IllegalArgumentException("Can't determine PhotometricInterpretation for color model: " + colorModel);
     }
 
-    private short[] createColorMap(final IndexColorModel colorModel) {
+    private short[] createColorMap(final IndexColorModel colorModel, final int sampleSize) {
         // TIFF6.pdf p. 23:
         // A TIFF color map is stored as type SHORT, count = 3 * (2^BitsPerSample)
         // "In a TIFF ColorMap, all the Red values come first, followed by the Green values, then the Blue values.
         // In the ColorMap, black is represented by 0,0,0 and white is represented by 65535, 65535, 65535."
-        short[] colorMap = new short[(int) (3 * Math.pow(2, colorModel.getPixelSize()))];
+        short[] colorMap = new short[(int) (3 * Math.pow(2, sampleSize))];
 
         for (int i = 0; i < colorModel.getMapSize(); i++) {
             int color = colorModel.getRGB(i);
@@ -650,14 +660,14 @@ public final class TIFFImageWriter extends ImageWriterBase {
         return shorts;
     }
 
-    private void writeImageData(DataOutput stream, RenderedImage renderedImage, int numComponents, int[] bandOffsets, int[] bitOffsets) throws IOException {
+    private void writeImageData(DataOutput stream, int imageIndex, RenderedImage renderedImage, int numComponents, int[] bandOffsets, int[] bitOffsets) throws IOException {
         // Store 3BYTE, 4BYTE as is (possibly need to re-arrange to RGB order)
         // Store INT_RGB as 3BYTE, INT_ARGB as 4BYTE?, INT_ABGR must be re-arranged
         // Store IndexColorModel as is
         // Store BYTE_GRAY as is
         // Store USHORT_GRAY as is
 
-        processImageStarted(0);
+        processImageStarted(imageIndex);
 
         final int minTileY = renderedImage.getMinTileY();
         final int maxYTiles = minTileY + renderedImage.getNumYTiles();
@@ -843,7 +853,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
             }
 
             // TODO: Report better progress
-            processImageProgress((100f * yTile) / maxYTiles);
+            processImageProgress((100f * (yTile + 1)) / maxYTiles);
         }
 
         if (stream instanceof DataOutputStream) {
@@ -918,8 +928,17 @@ public final class TIFFImageWriter extends ImageWriterBase {
             }
         }
 
-        // TODO: Set values from imageType
-        entries.put(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, new TIFFEntry(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, TIFF.TYPE_SHORT, getPhotometricInterpretation(imageType.getColorModel())));
+        int compression;
+        if ((param == null || param.getCompressionMode() == TIFFImageWriteParam.MODE_COPY_FROM_METADATA)
+                && ifd != null && ifd.getEntryById(TIFF.TAG_COMPRESSION) != null) {
+            compression = ((Number) ifd.getEntryById(TIFF.TAG_COMPRESSION).getValue()).intValue();
+        }
+        else {
+            compression = TIFFImageWriteParam.getCompressionType(param);
+        }
+
+        int photometricInterpretation = getPhotometricInterpretation(imageType.getColorModel(), compression);
+        entries.put(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, new TIFFEntry(TIFF.TAG_PHOTOMETRIC_INTERPRETATION, TIFF.TYPE_SHORT, photometricInterpretation));
 
         // TODO: Set values from param if != null + combined values...
 
@@ -950,13 +969,13 @@ public final class TIFFImageWriter extends ImageWriterBase {
 
     @Override
     public void prepareWriteSequence(IIOMetadata streamMetadata) throws IOException {
-        if (isWritingSequence) {
+        if (writingSequence) {
             throw new IllegalStateException("sequence writing has already been started!");
         }
 
-        // Ignore streamMetadata. ByteOrder is determined from OutputStream
         assertOutput();
-        isWritingSequence = true;
+
+        writingSequence = true;
         sequenceExifWriter = new EXIFWriter();
         sequenceExifWriter.writeTIFFHeader(imageOutput);
         sequenceLastIFDPos = imageOutput.getStreamPosition();
@@ -964,7 +983,7 @@ public final class TIFFImageWriter extends ImageWriterBase {
 
     @Override
     public void writeToSequence(IIOImage image, ImageWriteParam param) throws IOException {
-        if (!isWritingSequence) {
+        if (!writingSequence) {
             throw new IllegalStateException("prepareWriteSequence() must be called before writeToSequence()!");
         }
 
@@ -972,16 +991,17 @@ public final class TIFFImageWriter extends ImageWriterBase {
             imageOutput.flushBefore(sequenceLastIFDPos);
         }
 
-        sequenceLastIFDPos = writePage(image, param, sequenceExifWriter, sequenceLastIFDPos);
+        sequenceLastIFDPos = writePage(sequenceIndex++, image, param, sequenceExifWriter, sequenceLastIFDPos);
     }
 
     @Override
     public void endWriteSequence() throws IOException {
-        if (!isWritingSequence) {
+        if (!writingSequence) {
             throw new IllegalStateException("prepareWriteSequence() must be called before endWriteSequence()!");
         }
 
-        isWritingSequence = false;
+        writingSequence = false;
+        sequenceIndex = 0;
         sequenceExifWriter = null;
         sequenceLastIFDPos = -1;
         imageOutput.flush();
@@ -991,7 +1011,8 @@ public final class TIFFImageWriter extends ImageWriterBase {
     protected void resetMembers() {
         super.resetMembers();
 
-        isWritingSequence = false;
+        writingSequence = false;
+        sequenceIndex = 0;
         sequenceExifWriter = null;
         sequenceLastIFDPos = -1;
     }
@@ -1112,7 +1133,6 @@ public final class TIFFImageWriter extends ImageWriterBase {
 
         System.err.println("output.length: " + output.length());
 
-        // TODO: Support writing multipage TIFF
 //        ImageOutputStream stream = ImageIO.createImageOutputStream(output);
 //        try {
 //            writer.setOutput(stream);
@@ -1134,4 +1154,52 @@ public final class TIFFImageWriter extends ImageWriterBase {
 
         TIFFImageReader.showIt(read, output.getName());
     }
+
+    private class ListenerDelegate extends ProgressListenerBase implements IIOWriteWarningListener {
+        private final int imageIndex;
+
+        public ListenerDelegate(final int imageIndex) {
+            this.imageIndex = imageIndex;
+        }
+
+        @Override
+        public void imageComplete(ImageWriter source) {
+            processImageComplete();
+        }
+
+        @Override
+        public void imageProgress(ImageWriter source, float percentageDone) {
+            processImageProgress(percentageDone);
+        }
+
+        @Override
+        public void imageStarted(ImageWriter source, int imageIndex) {
+            processImageStarted(this.imageIndex);
+        }
+
+        @Override
+        public void thumbnailComplete(ImageWriter source) {
+            processThumbnailComplete();
+        }
+
+        @Override
+        public void thumbnailProgress(ImageWriter source, float percentageDone) {
+            processThumbnailProgress(percentageDone);
+        }
+
+        @Override
+        public void thumbnailStarted(ImageWriter source, int imageIndex, int thumbnailIndex) {
+            processThumbnailStarted(this.imageIndex, thumbnailIndex);
+        }
+
+        @Override
+        public void writeAborted(ImageWriter source) {
+            processWriteAborted();
+        }
+
+        @Override
+        public void warningOccurred(ImageWriter source, int imageIndex, String warning) {
+            processWarningOccurred(this.imageIndex, warning);
+        }
+    }
 }

imageio/imageio-tiff/src/test/java/com/twelvemonkeys/imageio/plugins/tiff/TIFFImageReaderTest.java

@@ -129,7 +129,9 @@ public class TIFFImageReaderTest extends ImageReaderAbstractTest<TIFFImageReader
                 new TestData(getClassLoaderResource("/tiff/depth/flower-separated-contig-16.tif"), new Dimension(73, 43)), // CMYK 16 bit/sample
                 // Separated (CMYK) Planar (PlanarConfiguration: 2)
                 new TestData(getClassLoaderResource("/tiff/depth/flower-separated-planar-08.tif"), new Dimension(73, 43)), // CMYK 8 bit/sample
-                new TestData(getClassLoaderResource("/tiff/depth/flower-separated-planar-16.tif"), new Dimension(73, 43))  // CMYK 16 bit/sample
+                new TestData(getClassLoaderResource("/tiff/depth/flower-separated-planar-16.tif"), new Dimension(73, 43)),  // CMYK 16 bit/sample
+                // Custom PIXTIFF ZIP (Compression: 50013)
+                new TestData(getClassLoaderResource("/tiff/pixtiff/40-8bit-gray-zip.tif"), new Dimension(801, 1313))  // ZIP Gray, 8 bit/sample
         );
     }
 
@@ -220,6 +222,25 @@ public class TIFFImageReaderTest extends ImageReaderAbstractTest<TIFFImageReader
         }
     }
 
+    @Test
+    public void testReadOldStyleJPEGInconsistentMetadata() throws IOException {
+        TestData testData = new TestData(getClassLoaderResource("/tiff/old-style-jpeg-inconsistent-metadata.tif"), new Dimension(2483, 3515));
+
+        try (ImageInputStream stream = testData.getInputStream()) {
+            TIFFImageReader reader = createReader();
+            reader.setInput(stream);
+
+            IIOReadWarningListener warningListener = mock(IIOReadWarningListener.class);
+            reader.addIIOReadWarningListener(warningListener);
+
+            BufferedImage image = reader.read(0);
+
+            assertNotNull(image);
+            assertEquals(testData.getDimension(0), new Dimension(image.getWidth(), image.getHeight()));
+            verify(warningListener, atLeastOnce()).warningOccurred(eq(reader), contains("metadata"));
+        }
+    }
+
     @Test
     public void testReadIncompatibleICCProfileIgnoredWithWarning() throws IOException {
         TestData testData = new TestData(getClassLoaderResource("/tiff/rgb-with-embedded-cmyk-icc.tif"), new Dimension(1500, 1500));

imageio/imageio-tiff/src/test/java/com/twelvemonkeys/imageio/plugins/tiff/TIFFImageWriterTest.java

@@ -36,10 +36,12 @@ import com.twelvemonkeys.imageio.metadata.exif.TIFF;
 import com.twelvemonkeys.imageio.stream.ByteArrayImageInputStream;
 import com.twelvemonkeys.imageio.util.ImageWriterAbstractTestCase;
 import com.twelvemonkeys.io.FastByteArrayOutputStream;
+import com.twelvemonkeys.io.NullOutputStream;
 import org.junit.Test;
 import org.w3c.dom.NodeList;
 
 import javax.imageio.*;
+import javax.imageio.event.IIOWriteProgressListener;
 import javax.imageio.metadata.IIOMetadata;
 import javax.imageio.metadata.IIOMetadataFormatImpl;
 import javax.imageio.metadata.IIOMetadataNode;
@@ -52,6 +54,8 @@ import java.io.*;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -59,6 +63,7 @@ import static com.twelvemonkeys.imageio.plugins.tiff.TIFFImageMetadataTest.creat
 import static com.twelvemonkeys.imageio.util.ImageReaderAbstractTest.assertRGBEquals;
 import static org.junit.Assert.*;
 import static org.junit.Assume.assumeNotNull;
+import static org.mockito.Mockito.*;
 
 /**
  * TIFFImageWriterTest
@@ -69,7 +74,7 @@ import static org.junit.Assume.assumeNotNull;
  */
 public class TIFFImageWriterTest extends ImageWriterAbstractTestCase {
 
-    public static final TIFFImageWriterSpi PROVIDER = new TIFFImageWriterSpi();
+    private static final TIFFImageWriterSpi PROVIDER = new TIFFImageWriterSpi();
 
     @Override
     protected ImageWriter createImageWriter() {
@@ -289,7 +294,27 @@ public class TIFFImageWriterTest extends ImageWriterAbstractTestCase {
         assertTrue("Writer should support sequence writing", writer.canWriteSequence());
     }
 
-    // TODO: Test Sequence writing without prepare/end sequence
+    @Test(expected = IllegalStateException.class)
+    public void testWriteSequenceWithoutPrepare() throws IOException {
+        ImageWriter writer = createImageWriter();
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+        try (ImageOutputStream output = ImageIO.createImageOutputStream(buffer)) {
+            writer.setOutput(output);
+            writer.writeToSequence(new IIOImage(new BufferedImage(10, 10, BufferedImage.TYPE_3BYTE_BGR), null, null), null);
+        }
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testEndSequenceWithoutPrepare() throws IOException {
+        ImageWriter writer = createImageWriter();
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+        try (ImageOutputStream output = ImageIO.createImageOutputStream(buffer)) {
+            writer.setOutput(output);
+            writer.endWriteSequence();
+        }
+    }
 
     @Test
     public void testWriteSequence() throws IOException {
@@ -361,6 +386,50 @@ public class TIFFImageWriterTest extends ImageWriterAbstractTestCase {
         }
     }
 
+    @Test
+    public void testWriteSequenceProgress() throws IOException {
+        BufferedImage[] images = new BufferedImage[] {
+                new BufferedImage(100, 100, BufferedImage.TYPE_INT_RGB),
+                new BufferedImage(110, 100, BufferedImage.TYPE_INT_RGB),
+                new BufferedImage(120, 100, BufferedImage.TYPE_INT_RGB)
+        };
+
+        ImageWriter writer = createImageWriter();
+        IIOWriteProgressListener progress = mock(IIOWriteProgressListener.class, "progress");
+        writer.addIIOWriteProgressListener(progress);
+
+        try (ImageOutputStream output = ImageIO.createImageOutputStream(new NullOutputStream())) {
+            writer.setOutput(output);
+
+            try {
+                writer.prepareWriteSequence(null);
+
+                for (int i = 0; i < images.length; i++) {
+                    reset(progress);
+
+                    ImageWriteParam param = writer.getDefaultWriteParam();
+
+                    if (i == images.length - 1) {
+                        // Make sure that the JPEG delegation outputs the correct indexes
+                        param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
+                        param.setCompressionType("JPEG");
+                    }
+
+                    writer.writeToSequence(new IIOImage(images[i], null, null), param);
+
+                    verify(progress, times(1)).imageStarted(writer, i);
+                    verify(progress, atLeastOnce()).imageProgress(eq(writer), anyFloat());
+                    verify(progress, times(1)).imageComplete(writer);
+                }
+
+                writer.endWriteSequence();
+            }
+            catch (IOException e) {
+                fail(e.getMessage());
+            }
+        }
+    }
+
     @Test
     public void testReadWriteRead1BitLZW() throws IOException {
         // Read original LZW compressed TIFF
@@ -789,4 +858,182 @@ public class TIFFImageWriterTest extends ImageWriterAbstractTestCase {
             }
         }
     }
+
+    @Test
+    public void testWriteStreamMetadataDefaultMM() throws IOException {
+        ImageWriter writer = createImageWriter();
+
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ImageOutputStream stream = ImageIO.createImageOutputStream(output)) {
+            stream.setByteOrder(ByteOrder.BIG_ENDIAN); // Should pass through
+            writer.setOutput(stream);
+
+            writer.write(null, new IIOImage(getTestData(0), null, null), null);
+        }
+
+        byte[] bytes = output.toByteArray();
+        assertArrayEquals(new byte[] {'M', 'M', 0, 42}, Arrays.copyOf(bytes, 4));
+    }
+
+    @Test
+    public void testWriteStreamMetadataDefaultII() throws IOException {
+        ImageWriter writer = createImageWriter();
+
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        try (ImageOutputStream stream = ImageIO.createImageOutputStream(output)) {
+            stream.setByteOrder(ByteOrder.LITTLE_ENDIAN); // Should pass through
+            writer.setOutput(stream);
+
+            writer.write(null, new IIOImage(getTestData(0), null, null), null);
+        }
+
+        byte[] bytes = output.toByteArray();
+        assertArrayEquals(new byte[] {'I', 'I', 42, 0}, Arrays.copyOf(bytes, 4));
+    }
+
+    @Test
+    public void testRewrite() throws IOException {
+        ImageWriter writer = createImageWriter();
+        ImageReader reader = ImageIO.getImageReader(writer);
+
+        List<URL> testData = Arrays.asList(
+                getClassLoaderResource("/tiff/pixtiff/17-tiff-binary-ccitt-group3.tif"),
+                getClassLoaderResource("/tiff/pixtiff/36-tiff-8-bit-gray-jpeg.tif"),
+                getClassLoaderResource("/tiff/pixtiff/51-tiff-24-bit-color-jpeg.tif"),
+                getClassLoaderResource("/tiff/pixtiff/58-plexustiff-binary-ccitt-group4.tif"),
+                getClassLoaderResource("/tiff/balloons.tif"),
+                getClassLoaderResource("/tiff/ColorCheckerCalculator.tif"),
+                getClassLoaderResource("/tiff/quad-jpeg.tif"),
+                getClassLoaderResource("/tiff/quad-lzw.tif"),
+                getClassLoaderResource("/tiff/bali.tif"),
+                getClassLoaderResource("/tiff/lzw-colormap-iiobe.tif"),
+                // TODO: FixMe for ColorMap + ExtraSamples (custom ColorModel)
+//                getClassLoaderResource("/tiff/colormap-with-extrasamples.tif"),
+
+                getClassLoaderResource("/tiff/depth/flower-minisblack-02.tif"),
+                getClassLoaderResource("/tiff/depth/flower-minisblack-04.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-minisblack-06.tif"),
+                getClassLoaderResource("/tiff/depth/flower-minisblack-08.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-minisblack-10.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-minisblack-12.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-minisblack-14.tif"),
+                getClassLoaderResource("/tiff/depth/flower-minisblack-16.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-minisblack-24.tif"),
+                getClassLoaderResource("/tiff/depth/flower-minisblack-32.tif"),
+
+                getClassLoaderResource("/tiff/depth/flower-palette-02.tif"),
+                getClassLoaderResource("/tiff/depth/flower-palette-04.tif"),
+                getClassLoaderResource("/tiff/depth/flower-palette-08.tif"),
+                getClassLoaderResource("/tiff/depth/flower-palette-16.tif"),
+
+                getClassLoaderResource("/tiff/depth/flower-rgb-contig-08.tif"),
+                // TODO: FixMe for RGB > 8 bits / sample
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-10.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-12.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-14.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-16.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-24.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-contig-32.tif"),
+
+                getClassLoaderResource("/tiff/depth/flower-rgb-planar-08.tif"),
+                // TODO: FixMe for planar RGB > 8 bits / sample
+//                getClassLoaderResource("/tiff/depth/flower-rgb-planar-10.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-planar-12.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-planar-14.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-planar-16.tif"),
+//                getClassLoaderResource("/tiff/depth/flower-rgb-planar-24.tif"),
+
+                getClassLoaderResource("/tiff/scan-mono-iccgray.tif"),
+                getClassLoaderResource("/tiff/old-style-jpeg-inconsistent-metadata.tif"),
+                getClassLoaderResource("/tiff/ccitt/group3_1d.tif"),
+                getClassLoaderResource("/tiff/ccitt/group3_2d.tif"),
+                getClassLoaderResource("/tiff/ccitt/group3_1d_fill.tif"),
+                getClassLoaderResource("/tiff/ccitt/group3_2d_fill.tif"),
+                getClassLoaderResource("/tiff/ccitt/group4.tif")
+        );
+
+        for (URL url : testData) {
+            ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+            try (ImageInputStream input = ImageIO.createImageInputStream(url);
+                 ImageOutputStream stream = ImageIO.createImageOutputStream(output)) {
+                reader.setInput(input);
+                writer.setOutput(stream);
+
+                List<ImageInfo> infos = new ArrayList<>(20);
+
+                writer.prepareWriteSequence(null);
+
+                for (int i = 0; i < reader.getNumImages(true); i++) {
+                    IIOImage image = reader.readAll(i, null);
+
+                    // If compression is Old JPEG, rewrite as JPEG
+                    // Normally, use the getAsTree method, but we don't care here if we are tied to our impl
+                    TIFFImageMetadata metadata = (TIFFImageMetadata) image.getMetadata();
+                    Directory ifd = metadata.getIFD();
+                    Entry compressionEntry = ifd.getEntryById(TIFF.TAG_COMPRESSION);
+
+                    int compression = compressionEntry != null ? ((Number) compressionEntry.getValue()).intValue() : TIFFBaseline.COMPRESSION_NONE;
+
+                    infos.add(new ImageInfo(image.getRenderedImage().getWidth(), image.getRenderedImage().getHeight(), compression));
+
+                    ImageWriteParam param = writer.getDefaultWriteParam();
+
+                    if (compression == TIFFExtension.COMPRESSION_OLD_JPEG) {
+                        param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); // Override the copy from metadata
+                        param.setCompressionType("JPEG");
+                    }
+
+                    writer.writeToSequence(image, param);
+                }
+
+                writer.endWriteSequence();
+
+//                File tempFile = File.createTempFile("foo-", ".tif");
+//                System.err.println("open " + tempFile.getAbsolutePath());
+//                FileUtil.write(tempFile, output.toByteArray());
+
+                try (ImageInputStream inputAfter = new ByteArrayImageInputStream(output.toByteArray())) {
+                    reader.setInput(inputAfter);
+
+                    int numImages = reader.getNumImages(true);
+
+                    assertEquals("Number of pages differs from original", infos.size(), numImages);
+
+                    for (int i = 0; i < numImages; i++) {
+                        IIOImage after = reader.readAll(i, null);
+                        ImageInfo info = infos.get(i);
+
+                        TIFFImageMetadata afterMetadata = (TIFFImageMetadata) after.getMetadata();
+                        Directory afterIfd = afterMetadata.getIFD();
+                        Entry afterCompressionEntry = afterIfd.getEntryById(TIFF.TAG_COMPRESSION);
+
+                        if (info.compression == TIFFExtension.COMPRESSION_OLD_JPEG) {
+                            // Should rewrite this from old-style to new style
+                            assertEquals("Old JPEG compression not rewritten as JPEG", TIFFExtension.COMPRESSION_JPEG, ((Number) afterCompressionEntry.getValue()).intValue());
+                        }
+                        else {
+                            assertEquals("Compression differs from original", info.compression, ((Number) afterCompressionEntry.getValue()).intValue());
+                        }
+
+                        assertEquals("Image width differs from original", info.width, after.getRenderedImage().getWidth());
+                        assertEquals("Image height differs from original", info.height, after.getRenderedImage().getHeight());
+                    }
+                }
+            }
+        }
+    }
+
+    private class ImageInfo {
+        final int width;
+        final int height;
+
+        final int compression;
+
+        private ImageInfo(int width, int height, int compression) {
+            this.width = width;
+            this.height = height;
+            this.compression = compression;
+        }
+    }
 }

imageio/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>com.twelvemonkeys</groupId>
         <artifactId>twelvemonkeys</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.twelvemonkeys.imageio</groupId>
@@ -101,6 +101,13 @@
             <version>1.8.5</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <version>2.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>

pom.xml

@@ -8,7 +8,7 @@
     </parent>
     <groupId>com.twelvemonkeys</groupId>
     <artifactId>twelvemonkeys</artifactId>
-    <version>3.3</version>
+    <version>3.3.3-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>Twelvemonkeys</name>
 

servlet/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>com.twelvemonkeys</groupId>
         <artifactId>twelvemonkeys</artifactId>
-        <version>3.3</version>
+        <version>3.3.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>