Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
markov
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Commits
Open sidebar
Patrick Friedel
markov
Commits
0d976c4d
Commit
0d976c4d
authored
Mar 13, 2018
by
Hut
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
some cleanup
parent
6613e459
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
23 additions
and
20 deletions
+23
-20
Builder.java
src/main/java/markov/Builder.java
+1
-1
Data.java
src/main/java/markov/Data.java
+1
-1
Glyph.java
src/main/java/markov/Glyph.java
+2
-4
Lookup.java
src/main/java/markov/Lookup.java
+1
-1
Parser.java
src/main/java/markov/Parser.java
+2
-1
Renderer.java
src/main/java/markov/Renderer.java
+1
-1
Token.java
src/main/java/markov/Token.java
+2
-2
Tokenizer.java
src/main/java/markov/Tokenizer.java
+8
-4
ByteHuffmanCodeBuilder.java
src/main/java/markov/huffman/ByteHuffmanCodeBuilder.java
+2
-2
HuffmanCodeBuilder.java
src/main/java/markov/huffman/HuffmanCodeBuilder.java
+1
-1
Utils.java
src/main/java/markov/stuff/Utils.java
+2
-2
No files found.
src/main/java/markov/Builder.java
View file @
0d976c4d
...
...
@@ -20,7 +20,7 @@ public class Builder {
}
public
Sentence
random
()
{
return
produce
(
p
->
nextRandom
(
p
)
);
return
produce
(
this
::
nextRandom
);
}
public
Sentence
average
()
{
...
...
src/main/java/markov/Data.java
View file @
0d976c4d
...
...
@@ -71,6 +71,6 @@ public class Data implements Serializable{
}
public
void
finish
()
{
data
.
values
().
forEach
(
l
->
l
.
finishCollection
()
);
data
.
values
().
forEach
(
Lookup:
:
finishCollection
);
}
}
src/main/java/markov/Glyph.java
View file @
0d976c4d
...
...
@@ -31,10 +31,8 @@ public class Glyph implements Serializable {
Glyph
glyph
=
(
Glyph
)
o
;
if
(
type
!=
glyph
.
type
)
{
return
false
;
}
return
content
!=
null
?
content
.
equals
(
glyph
.
content
)
:
glyph
.
content
==
null
;
return
type
==
glyph
.
type
&&
(
content
!=
null
?
content
.
equals
(
glyph
.
content
)
:
glyph
.
content
==
null
);
}
@Override
...
...
src/main/java/markov/Lookup.java
View file @
0d976c4d
...
...
@@ -10,7 +10,7 @@ import java.util.stream.Collectors;
public
class
Lookup
implements
Serializable
{
private
final
LinkedHashMap
<
Token
,
Integer
>
tokens
=
new
LinkedHashMap
<
Token
,
Integer
>();
private
final
LinkedHashMap
<
Token
,
Integer
>
tokens
=
new
LinkedHashMap
<>();
private
Decission
[]
finalData
;
private
long
[]
finishedSums
;
private
boolean
isFinishedCollecting
=
false
;
...
...
src/main/java/markov/Parser.java
View file @
0d976c4d
...
...
@@ -25,6 +25,7 @@ public class Parser {
c
.
reset
();
}
});
return
collectors
.
stream
().
collect
(
Collectors
.
toMap
(
c
->
c
.
getPrefixLength
(),
c
->
c
.
finishAndGetData
()));
return
collectors
.
stream
().
collect
(
Collectors
.
toMap
(
Collector:
:
getPrefixLength
,
Collector:
:
finishAndGetData
));
}
}
src/main/java/markov/Renderer.java
View file @
0d976c4d
...
...
@@ -71,7 +71,7 @@ public class Renderer {
sb
.
append
(
d
.
getToken
().
render
(
options
.
prefix
));
}
if
(
options
.
propability
)
{
sb
.
append
(
" - "
+
sentence
.
propability
());
sb
.
append
(
" - "
).
append
(
sentence
.
propability
());
}
return
sb
.
toString
();
}
...
...
src/main/java/markov/Token.java
View file @
0d976c4d
...
...
@@ -42,8 +42,8 @@ public class Token implements Serializable, Comparable<Token> {
Token
token
=
(
Token
)
o
;
if
(
content
!=
null
?
!
content
.
equals
(
token
.
content
)
:
token
.
content
!=
null
)
return
false
;
return
type
==
token
.
type
;
return
(
content
!=
null
?
content
.
equals
(
token
.
content
)
:
token
.
content
==
null
)
&&
type
==
token
.
type
;
}
private
final
int
hashCode
;
...
...
src/main/java/markov/Tokenizer.java
View file @
0d976c4d
...
...
@@ -10,11 +10,15 @@ public class Tokenizer {
public
Stream
<
Stream
<
Token
>>
tokenizeCombined
(
Stream
<
Stream
<
String
>>
input
)
{
return
input
.
map
(
stringStream
->
putMarkers
(
combineToTokens
(
stringStream
.
flatMap
(
s
->
s
.
codePoints
().
boxed
().
map
(
integer
->
glyphFromCodePoint
(
integer
))))));
return
input
.
map
(
stringStream
->
putMarkers
(
combineToTokens
(
stringStream
.
flatMap
(
s
->
s
.
codePoints
().
boxed
().
map
(
this
::
glyphFromCodePoint
)))));
}
public
Stream
<
Stream
<
Token
>>
tokenize
(
Stream
<
String
>
input
)
{
return
input
.
map
(
s
->
putMarkers
(
combineToTokens
(
s
.
codePoints
().
boxed
().
map
(
integer
->
glyphFromCodePoint
(
integer
)))));
return
input
.
map
(
s
->
putMarkers
(
combineToTokens
(
s
.
codePoints
().
boxed
().
map
(
this
::
glyphFromCodePoint
))));
}
private
Glyph
glyphFromCodePoint
(
int
codePoint
)
{
...
...
@@ -58,11 +62,11 @@ public class Tokenizer {
}
else
{
return
Stream
.
empty
();
}
}).
map
(
l
->
getTokenFromGlyphs
(
l
)
);
}).
map
(
this
::
getTokenFromGlyphs
);
}
private
Token
getTokenFromGlyphs
(
List
<
Glyph
>
l
)
{
String
content
=
l
.
stream
().
map
(
glyph
->
glyph
.
getContent
()
).
collect
(
Collectors
.
joining
());
String
content
=
l
.
stream
().
map
(
Glyph:
:
getContent
).
collect
(
Collectors
.
joining
());
Glyph
.
Type
type
=
l
.
get
(
0
).
getType
();
return
new
Token
(
content
,
type
);
}
...
...
src/main/java/markov/huffman/ByteHuffmanCodeBuilder.java
View file @
0d976c4d
...
...
@@ -9,7 +9,7 @@ import java.util.function.Supplier;
public
class
ByteHuffmanCodeBuilder
<
ContentType
,
FrequencyType
extends
FrequenceType
<
FrequencyType
>>
extends
HuffmanCodeBuilder
<
ContentType
,
List
<
Boolean
>,
Boolean
,
FrequencyType
>
{
private
static
final
Supplier
<
List
<
Boolean
>>
rootCodeSupplier
=
()
->
new
ArrayList
<>()
;
private
static
final
Supplier
<
List
<
Boolean
>>
rootCodeSupplier
=
ArrayList:
:
new
;
private
static
final
Supplier
<
Boolean
>
leftGlyph
=
()
->
false
;
private
static
final
Supplier
<
Boolean
>
rightGlyph
=
()
->
true
;
private
static
final
BiFunction
<
List
<
Boolean
>,
Boolean
,
List
<
Boolean
>>
combiner
=
(
l
,
g
)
->
{
...
...
@@ -17,7 +17,7 @@ public class ByteHuffmanCodeBuilder<ContentType, FrequencyType extends Frequence
l
.
add
(
g
);
return
l
;
};
private
static
final
Function
<
List
<
Boolean
>,
Iterator
<
Boolean
>>
splitter
=
l
->
l
.
iterator
()
;
private
static
final
Function
<
List
<
Boolean
>,
Iterator
<
Boolean
>>
splitter
=
List:
:
iterator
;
public
ByteHuffmanCodeBuilder
()
{
...
...
src/main/java/markov/huffman/HuffmanCodeBuilder.java
View file @
0d976c4d
...
...
@@ -55,7 +55,7 @@ public class HuffmanCodeBuilder<ContentType, SequenceType, GlyphType, FrequencyT
PriorityQueue
<
HuffmanTree
<
ContentType
,
SequenceType
,
FrequencyType
>>
trees
=
new
PriorityQueue
<>();
for
(
Map
.
Entry
<
ContentType
,
FrequencyType
>
e
:
frequencies
.
entrySet
())
{
if
(
e
.
getValue
().
isGreaterZero
())
trees
.
offer
(
new
HuffmanLeaf
<
ContentType
,
SequenceType
,
FrequencyType
>(
e
.
getValue
(),
e
.
getKey
()));
trees
.
offer
(
new
HuffmanLeaf
<>(
e
.
getValue
(),
e
.
getKey
()));
}
while
(
trees
.
size
()
>
1
)
{
...
...
src/main/java/markov/stuff/Utils.java
View file @
0d976c4d
...
...
@@ -109,8 +109,8 @@ public class Utils {
public
static
Map
<
Integer
,
Data
>
createDataMap
(
int
maxPrefix
,
File
parent
)
{
System
.
out
.
println
(
"generating data map..."
);
Stream
<
String
>
mails
=
Arrays
.
stream
(
parent
.
listFiles
()).
limit
(
500
).
map
(
file
->
file
.
toPath
()).
map
(
path
->
{
Stream
<
String
>
mails
=
Arrays
.
stream
(
parent
.
listFiles
()).
limit
(
500
).
map
(
File:
:
toPath
).
map
(
path
->
{
try
{
MimeMessage
m
=
new
MimeMessage
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment