added stream style parser

src/main/java/markov/Data.java

@@ -25,7 +25,7 @@ public class Data {
 	}
 
 	public Lookup info(String... s) {
-		return data.get(new Prefix(Arrays.stream(s).map(t -> new Token(t))
+		return data.get(new Prefix(Arrays.stream(s).map(t -> new Token(t, Glyph.Type.word))
 				.collect(Collectors.toList())));
 	}
 

src/main/java/markov/Glyph.java

@@ -2,9 +2,12 @@ package markov;
 
 public class Glyph {
     public enum Type {
-        word, interpunction, control, whitespace
+        word, interpunction, control, whitespace, empty
     }
 
+    public static final Glyph Start = new Glyph(Type.control, "START");
+    public static final Glyph End = new Glyph(Type.control, "END");
+
     private Type type;
     private String content;
 
@@ -29,7 +32,11 @@ public class Glyph {
 
         Glyph glyph = (Glyph) o;
 
-        if (type != glyph.type) return false;
+        if (type != glyph.type) {
+            return false;
+        }else if(type == Type.control){
+            return this == o;
+        }
         return content != null ? content.equals(glyph.content) : glyph.content == null;
     }
 

src/main/java/markov/ParserStreamStyle.java

 package markov;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 public class ParserStreamStyle {
 
+    public Stream<Token> collect(Stream<Glyph> glyphs) {
+        Data data = new Data();
+
+        final Container[] previous = new Container[]{null};
+        return glyphs.map(g -> {
+            Container o = new Container(g);
+            o.p = previous[0];
+            previous[0] = o;
+            return o;
+        }).flatMap(go -> {
+            if (go.p != null && go.p.self != null && !go.self.getType().equals(go.p.self.getType())) {
+                List<Glyph> containers = new ArrayList<>();
+                Container c = go;
+                do {
+                    c = c.p;
+                    containers.add(0, c.self);
+                }
+                while (c != null && c.p != null && c.p.self != null  &&
+                        c.self.getType().equals(c.p.self.getType()));
+                return Stream.<List<Glyph>>builder().add(containers).build();
+            } else {
+                return Stream.empty();
+            }
+        }).map(l -> {
+            String content = l.stream().map(glyph -> glyph.getContent()).collect(Collectors.joining());
+            Glyph.Type type = l.get(0).getType();
+            return new Token(content, type);
+        });
+    }
+
+    private static class Container {
+        Container(Glyph self) {
+            this.self = self;
+        }
+
+        final Glyph self;
+        Container p;
+    }
+
+    ;
 }

src/main/java/markov/Parser_first_version.java

@@ -32,7 +32,7 @@ public class Parser_first_version {
             tokens[i] = Token.START;
         }
         for (; i < strings.length + prefix_length; i++) {
-            tokens[i] = new Token(strings[i - prefix_length]);
+            tokens[i] = new Token(strings[i - prefix_length], Glyph.Type.word);
         }
         tokens[i] = Token.END;
         return tokens;

src/main/java/markov/Token.java

@@ -7,11 +7,15 @@ public class Token {
 	protected static final Token EMPTY = new SpecialToken();
 
 	public Token(String content) {
+	    this(content, Glyph.Type.word);
+    }
+	public Token(String content, Glyph.Type type) {
 		super();
 		this.content = content;
 	}
 
-	String content;
+	private String content;
+	private Glyph.Type type;
 
 	@Override
 	public String toString() {
@@ -19,53 +23,39 @@ public class Token {
 			return "Token <END>";
 		}
 		if (this == START) {
-			return "TOKEN <START>";
+			return "TOKEN <Start>";
 		}
 		if (this == EMPTY)
 			return "TOKEN <empty>";
 		return "T=[" + content + "]";
 	}
 
-	@Override
-	public int hashCode() {
-		final int prime = 31;
-		int result = 1;
-		result = prime * result + ((content == null) ? 0 : content.hashCode());
-		return result;
-	}
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
 
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj) {
-			return true;
-		}
-		if (obj == null)
-			return false;
-		if (this == START)
-			return false;
-		if (this == END)
-			return false;
-		if (this == EMPTY)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-		Token other = (Token) obj;
-		if (content == null) {
-			if (other.content != null)
-				return false;
-		} else if (!content.equals(other.content))
-			return false;
-		return true;
-	}
+        Token token = (Token) o;
+
+        if (content != null ? !content.equals(token.content) : token.content != null) return false;
+        return type == token.type;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = content != null ? content.hashCode() : 0;
+        result = 31 * result + (type != null ? type.hashCode() : 0);
+        return result;
+    }
 
-	public String render(String prefix) {
+    public String render(String prefix) {
 		return prefix + content;
 	}
 
 	private static class SpecialToken extends Token {
 
 		public SpecialToken() {
-			super("");
+			super("", Glyph.Type.control);
 		}
 
 		@Override

src/test/java/markov/GlyphTest.java

+package markov;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class GlyphTest {
+
+    @Test
+    public void testEquals() {
+        Glyph a = new Glyph(Glyph.Type.word, "a");
+        Glyph a2 = new Glyph(Glyph.Type.word, "a");
+        Glyph w = new Glyph(Glyph.Type.whitespace, " ");
+        Glyph c = new Glyph(Glyph.Type.control, "");
+
+        assertEquals(a, a2);
+        assertNotEquals(a, w);
+        assertNotEquals(a, c);
+        assertNotEquals(Glyph.End, c);
+        assertNotEquals(Glyph.Start, c);
+        assertNotEquals(Glyph.Start, Glyph.End);
+        assertEquals(Glyph.Start, Glyph.Start);
+        assertEquals(Glyph.End, Glyph.End);
+
+    }
+}
\ No newline at end of file

src/test/java/markov/ParserStreamStyleTest.java

+package markov;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+class ParserStreamStyleTest {
+
+    @Test
+    void tests() {
+        ParserStreamStyle parserStreamStyle = new ParserStreamStyle();
+        Assertions.assertIterableEquals(
+                Arrays.asList(
+                        new Token(Glyph.Start.getContent(), Glyph.Type.control),
+                        new Token("ab"),
+                        new Token(" ", Glyph.Type.whitespace),
+                        new Token("c"),
+                        new Token(Glyph.End.getContent(), Glyph.Type.control)),
+                parserStreamStyle.collect(Stream.of(Glyph.Start,
+                        new Glyph(Glyph.Type.word, "a"),
+                        new Glyph(Glyph.Type.word, "b"),
+                        new Glyph(Glyph.Type.whitespace, " "),
+                        new Glyph(Glyph.Type.word, "c"),
+                        Glyph.End, new Glyph(Glyph.Type.empty, "EMPTY")))
+                        .collect(Collectors.toList()));
+
+
+    }
+}
\ No newline at end of file