Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
markov
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Commits
Open sidebar
Patrick Friedel
markov
Commits
0d976c4d
Commit
0d976c4d
authored
Mar 13, 2018
by
Hut
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
some cleanup
parent
6613e459
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
23 additions
and
20 deletions
+23
-20
Builder.java
src/main/java/markov/Builder.java
+1
-1
Data.java
src/main/java/markov/Data.java
+1
-1
Glyph.java
src/main/java/markov/Glyph.java
+2
-4
Lookup.java
src/main/java/markov/Lookup.java
+1
-1
Parser.java
src/main/java/markov/Parser.java
+2
-1
Renderer.java
src/main/java/markov/Renderer.java
+1
-1
Token.java
src/main/java/markov/Token.java
+2
-2
Tokenizer.java
src/main/java/markov/Tokenizer.java
+8
-4
ByteHuffmanCodeBuilder.java
src/main/java/markov/huffman/ByteHuffmanCodeBuilder.java
+2
-2
HuffmanCodeBuilder.java
src/main/java/markov/huffman/HuffmanCodeBuilder.java
+1
-1
Utils.java
src/main/java/markov/stuff/Utils.java
+2
-2
No files found.
src/main/java/markov/Builder.java
View file @
0d976c4d
...
@@ -20,7 +20,7 @@ public class Builder {
...
@@ -20,7 +20,7 @@ public class Builder {
}
}
public
Sentence
random
()
{
public
Sentence
random
()
{
return
produce
(
p
->
nextRandom
(
p
)
);
return
produce
(
this
::
nextRandom
);
}
}
public
Sentence
average
()
{
public
Sentence
average
()
{
...
...
src/main/java/markov/Data.java
View file @
0d976c4d
...
@@ -71,6 +71,6 @@ public class Data implements Serializable{
...
@@ -71,6 +71,6 @@ public class Data implements Serializable{
}
}
public
void
finish
()
{
public
void
finish
()
{
data
.
values
().
forEach
(
l
->
l
.
finishCollection
()
);
data
.
values
().
forEach
(
Lookup:
:
finishCollection
);
}
}
}
}
src/main/java/markov/Glyph.java
View file @
0d976c4d
...
@@ -31,10 +31,8 @@ public class Glyph implements Serializable {
...
@@ -31,10 +31,8 @@ public class Glyph implements Serializable {
Glyph
glyph
=
(
Glyph
)
o
;
Glyph
glyph
=
(
Glyph
)
o
;
if
(
type
!=
glyph
.
type
)
{
return
type
==
glyph
.
type
&&
(
content
!=
null
?
content
.
equals
(
glyph
.
content
)
:
return
false
;
glyph
.
content
==
null
);
}
return
content
!=
null
?
content
.
equals
(
glyph
.
content
)
:
glyph
.
content
==
null
;
}
}
@Override
@Override
...
...
src/main/java/markov/Lookup.java
View file @
0d976c4d
...
@@ -10,7 +10,7 @@ import java.util.stream.Collectors;
...
@@ -10,7 +10,7 @@ import java.util.stream.Collectors;
public
class
Lookup
implements
Serializable
{
public
class
Lookup
implements
Serializable
{
private
final
LinkedHashMap
<
Token
,
Integer
>
tokens
=
new
LinkedHashMap
<
Token
,
Integer
>();
private
final
LinkedHashMap
<
Token
,
Integer
>
tokens
=
new
LinkedHashMap
<>();
private
Decission
[]
finalData
;
private
Decission
[]
finalData
;
private
long
[]
finishedSums
;
private
long
[]
finishedSums
;
private
boolean
isFinishedCollecting
=
false
;
private
boolean
isFinishedCollecting
=
false
;
...
...
src/main/java/markov/Parser.java
View file @
0d976c4d
...
@@ -25,6 +25,7 @@ public class Parser {
...
@@ -25,6 +25,7 @@ public class Parser {
c
.
reset
();
c
.
reset
();
}
}
});
});
return
collectors
.
stream
().
collect
(
Collectors
.
toMap
(
c
->
c
.
getPrefixLength
(),
c
->
c
.
finishAndGetData
()));
return
collectors
.
stream
().
collect
(
Collectors
.
toMap
(
Collector:
:
getPrefixLength
,
Collector:
:
finishAndGetData
));
}
}
}
}
src/main/java/markov/Renderer.java
View file @
0d976c4d
...
@@ -71,7 +71,7 @@ public class Renderer {
...
@@ -71,7 +71,7 @@ public class Renderer {
sb
.
append
(
d
.
getToken
().
render
(
options
.
prefix
));
sb
.
append
(
d
.
getToken
().
render
(
options
.
prefix
));
}
}
if
(
options
.
propability
)
{
if
(
options
.
propability
)
{
sb
.
append
(
" - "
+
sentence
.
propability
());
sb
.
append
(
" - "
).
append
(
sentence
.
propability
());
}
}
return
sb
.
toString
();
return
sb
.
toString
();
}
}
...
...
src/main/java/markov/Token.java
View file @
0d976c4d
...
@@ -42,8 +42,8 @@ public class Token implements Serializable, Comparable<Token> {
...
@@ -42,8 +42,8 @@ public class Token implements Serializable, Comparable<Token> {
Token
token
=
(
Token
)
o
;
Token
token
=
(
Token
)
o
;
if
(
content
!=
null
?
!
content
.
equals
(
token
.
content
)
:
token
.
content
!=
null
)
return
false
;
return
(
content
!=
null
?
content
.
equals
(
token
.
content
)
:
token
.
content
==
null
)
&&
return
type
==
token
.
type
;
type
==
token
.
type
;
}
}
private
final
int
hashCode
;
private
final
int
hashCode
;
...
...
src/main/java/markov/Tokenizer.java
View file @
0d976c4d
...
@@ -10,11 +10,15 @@ public class Tokenizer {
...
@@ -10,11 +10,15 @@ public class Tokenizer {
public
Stream
<
Stream
<
Token
>>
tokenizeCombined
(
Stream
<
Stream
<
String
>>
input
)
{
public
Stream
<
Stream
<
Token
>>
tokenizeCombined
(
Stream
<
Stream
<
String
>>
input
)
{
return
input
.
map
(
stringStream
->
putMarkers
(
combineToTokens
(
stringStream
.
flatMap
(
s
->
s
.
codePoints
().
boxed
().
map
(
integer
->
glyphFromCodePoint
(
integer
))))));
return
input
.
map
(
stringStream
->
putMarkers
(
combineToTokens
(
stringStream
.
flatMap
(
s
->
s
.
codePoints
().
boxed
().
map
(
this
::
glyphFromCodePoint
)))));
}
}
public
Stream
<
Stream
<
Token
>>
tokenize
(
Stream
<
String
>
input
)
{
public
Stream
<
Stream
<
Token
>>
tokenize
(
Stream
<
String
>
input
)
{
return
input
.
map
(
s
->
putMarkers
(
combineToTokens
(
s
.
codePoints
().
boxed
().
map
(
integer
->
glyphFromCodePoint
(
integer
)))));
return
input
.
map
(
s
->
putMarkers
(
combineToTokens
(
s
.
codePoints
().
boxed
().
map
(
this
::
glyphFromCodePoint
))));
}
}
private
Glyph
glyphFromCodePoint
(
int
codePoint
)
{
private
Glyph
glyphFromCodePoint
(
int
codePoint
)
{
...
@@ -58,11 +62,11 @@ public class Tokenizer {
...
@@ -58,11 +62,11 @@ public class Tokenizer {
}
else
{
}
else
{
return
Stream
.
empty
();
return
Stream
.
empty
();
}
}
}).
map
(
l
->
getTokenFromGlyphs
(
l
)
);
}).
map
(
this
::
getTokenFromGlyphs
);
}
}
private
Token
getTokenFromGlyphs
(
List
<
Glyph
>
l
)
{
private
Token
getTokenFromGlyphs
(
List
<
Glyph
>
l
)
{
String
content
=
l
.
stream
().
map
(
glyph
->
glyph
.
getContent
()
).
collect
(
Collectors
.
joining
());
String
content
=
l
.
stream
().
map
(
Glyph:
:
getContent
).
collect
(
Collectors
.
joining
());
Glyph
.
Type
type
=
l
.
get
(
0
).
getType
();
Glyph
.
Type
type
=
l
.
get
(
0
).
getType
();
return
new
Token
(
content
,
type
);
return
new
Token
(
content
,
type
);
}
}
...
...
src/main/java/markov/huffman/ByteHuffmanCodeBuilder.java
View file @
0d976c4d
...
@@ -9,7 +9,7 @@ import java.util.function.Supplier;
...
@@ -9,7 +9,7 @@ import java.util.function.Supplier;
public
class
ByteHuffmanCodeBuilder
<
ContentType
,
FrequencyType
extends
FrequenceType
<
FrequencyType
>>
extends
HuffmanCodeBuilder
<
ContentType
,
List
<
Boolean
>,
Boolean
,
FrequencyType
>
{
public
class
ByteHuffmanCodeBuilder
<
ContentType
,
FrequencyType
extends
FrequenceType
<
FrequencyType
>>
extends
HuffmanCodeBuilder
<
ContentType
,
List
<
Boolean
>,
Boolean
,
FrequencyType
>
{
private
static
final
Supplier
<
List
<
Boolean
>>
rootCodeSupplier
=
()
->
new
ArrayList
<>()
;
private
static
final
Supplier
<
List
<
Boolean
>>
rootCodeSupplier
=
ArrayList:
:
new
;
private
static
final
Supplier
<
Boolean
>
leftGlyph
=
()
->
false
;
private
static
final
Supplier
<
Boolean
>
leftGlyph
=
()
->
false
;
private
static
final
Supplier
<
Boolean
>
rightGlyph
=
()
->
true
;
private
static
final
Supplier
<
Boolean
>
rightGlyph
=
()
->
true
;
private
static
final
BiFunction
<
List
<
Boolean
>,
Boolean
,
List
<
Boolean
>>
combiner
=
(
l
,
g
)
->
{
private
static
final
BiFunction
<
List
<
Boolean
>,
Boolean
,
List
<
Boolean
>>
combiner
=
(
l
,
g
)
->
{
...
@@ -17,7 +17,7 @@ public class ByteHuffmanCodeBuilder<ContentType, FrequencyType extends Frequence
...
@@ -17,7 +17,7 @@ public class ByteHuffmanCodeBuilder<ContentType, FrequencyType extends Frequence
l
.
add
(
g
);
l
.
add
(
g
);
return
l
;
return
l
;
};
};
private
static
final
Function
<
List
<
Boolean
>,
Iterator
<
Boolean
>>
splitter
=
l
->
l
.
iterator
()
;
private
static
final
Function
<
List
<
Boolean
>,
Iterator
<
Boolean
>>
splitter
=
List:
:
iterator
;
public
ByteHuffmanCodeBuilder
()
{
public
ByteHuffmanCodeBuilder
()
{
...
...
src/main/java/markov/huffman/HuffmanCodeBuilder.java
View file @
0d976c4d
...
@@ -55,7 +55,7 @@ public class HuffmanCodeBuilder<ContentType, SequenceType, GlyphType, FrequencyT
...
@@ -55,7 +55,7 @@ public class HuffmanCodeBuilder<ContentType, SequenceType, GlyphType, FrequencyT
PriorityQueue
<
HuffmanTree
<
ContentType
,
SequenceType
,
FrequencyType
>>
trees
=
new
PriorityQueue
<>();
PriorityQueue
<
HuffmanTree
<
ContentType
,
SequenceType
,
FrequencyType
>>
trees
=
new
PriorityQueue
<>();
for
(
Map
.
Entry
<
ContentType
,
FrequencyType
>
e
:
frequencies
.
entrySet
())
{
for
(
Map
.
Entry
<
ContentType
,
FrequencyType
>
e
:
frequencies
.
entrySet
())
{
if
(
e
.
getValue
().
isGreaterZero
())
if
(
e
.
getValue
().
isGreaterZero
())
trees
.
offer
(
new
HuffmanLeaf
<
ContentType
,
SequenceType
,
FrequencyType
>(
e
.
getValue
(),
e
.
getKey
()));
trees
.
offer
(
new
HuffmanLeaf
<>(
e
.
getValue
(),
e
.
getKey
()));
}
}
while
(
trees
.
size
()
>
1
)
{
while
(
trees
.
size
()
>
1
)
{
...
...
src/main/java/markov/stuff/Utils.java
View file @
0d976c4d
...
@@ -109,8 +109,8 @@ public class Utils {
...
@@ -109,8 +109,8 @@ public class Utils {
public
static
Map
<
Integer
,
Data
>
createDataMap
(
int
maxPrefix
,
File
parent
)
{
public
static
Map
<
Integer
,
Data
>
createDataMap
(
int
maxPrefix
,
File
parent
)
{
System
.
out
.
println
(
"generating data map..."
);
System
.
out
.
println
(
"generating data map..."
);
Stream
<
String
>
mails
=
Arrays
.
stream
(
parent
.
listFiles
()).
limit
(
500
).
map
(
Stream
<
String
>
mails
=
Arrays
.
stream
(
parent
.
listFiles
()).
limit
(
500
).
map
(
File:
:
toPath
).
map
file
->
file
.
toPath
()).
map
(
path
->
{
(
path
->
{
try
{
try
{
MimeMessage
m
=
new
MimeMessage
(
MimeMessage
m
=
new
MimeMessage
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment