Skip to content

Commit 50d1e51

Browse files
dmi3coderdmi3coder
authored
Merge pull request #41 from dmi3coder/develop
Proto serialization
2 parents 250d05d + b7e8a15 commit 50d1e51

20 files changed

+3057
-79
lines changed

‎build.gradle

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ publishing {
3333
gpr(MavenPublication) {
3434
groupId = 'tech.donau.behaiv'
3535
artifactId = 'behaiv-java'
36-
version = '0.4.8-alpha'
36+
version = '0.4.10-alpha'
3737
from(components.java)
3838
}
3939
}
@@ -49,7 +49,7 @@ repositories {
4949

5050
dependencies {
5151
// Protobuf
52-
compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.11.4'
52+
compile group: 'com.google.protobuf', name: 'protobuf-javalite', version: '3.11.4'
5353
// Swagger codegen dependencies
5454
swaggerCodegen 'io.swagger:swagger-codegen-cli:2.4.2' // Swagger Codegen V2
5555
// This dependency is exported to consumers, that is to say found on their compile classpath.
@@ -90,6 +90,11 @@ jacocoTestReport {
9090
xml.enabled = true
9191
csv.enabled = false
9292
}
93+
afterEvaluate {
94+
classDirectories.setFrom(files(classDirectories.files.collect {
95+
fileTree(dir: it, exclude: 'tech/donau/behaiv/proto**')
96+
}))
97+
}
9398
}
9499

95100
task proto(type: Exec) {

‎proto.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
protoc -I=. --java_out=./src/main/java ./behaiv.proto
1+
protoc -I=. --java_out=lite:./src/main/java ./behaiv.proto

‎src/main/java/de/dmi3y/behaiv/kernel/BaseKernel.java

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package de.dmi3y.behaiv.kernel;
2+
3+
import com.fasterxml.jackson.core.type.TypeReference;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.google.protobuf.CodedOutputStream;
6+
import de.dmi3y.behaiv.storage.BehaivStorage;
7+
import de.dmi3y.behaiv.tools.DataMappingUtils;
8+
import de.dmi3y.behaiv.tools.Pair;
9+
import tech.donau.behaiv.proto.PredictionSet;
10+
11+
import java.io.BufferedReader;
12+
import java.io.BufferedWriter;
13+
import java.io.File;
14+
import java.io.FileInputStream;
15+
import java.io.FileNotFoundException;
16+
import java.io.FileOutputStream;
17+
import java.io.FileReader;
18+
import java.io.FileWriter;
19+
import java.io.IOException;
20+
import java.util.ArrayList;
21+
import java.util.List;
22+
23+
public abstract class BaseKernel implements Kernel {
24+
25+
protected String id;
26+
protected Long treshold = 10L;
27+
protected ObjectMapper objectMapper;
28+
protected boolean partialFitAllowed = false;
29+
protected boolean alwaysKeepData = true;
30+
31+
public BaseKernel(String id) {
32+
this.id = id;
33+
objectMapper = new ObjectMapper();
34+
}
35+
36+
37+
@Override
38+
public void setId(String id) {
39+
this.id = id;
40+
}
41+
42+
43+
//list<features>, label
44+
protected List<Pair<List<Double>, String>> data = new ArrayList<>();
45+
46+
47+
@Override
48+
public void fit() {
49+
fit(this.data);
50+
}
51+
52+
@Override
53+
public void setTreshold(Long treshold) {
54+
this.treshold = treshold;
55+
}
56+
57+
@Override
58+
public boolean readyToPredict() {
59+
return data.size() > treshold;
60+
}
61+
62+
@Override
63+
public void update(List<Pair<List<Double>, String>> data) {
64+
}
65+
66+
@Override
67+
public boolean isPartialFitAllowed() {
68+
return partialFitAllowed;
69+
}
70+
71+
@Override
72+
public void updateSingle(List<Double> features, String label) {
73+
data.add(new Pair<>(features, label));
74+
}
75+
76+
@Override
77+
public boolean isAlwaysKeepData() {
78+
return alwaysKeepData;
79+
}
80+
81+
@Override
82+
public void setAlwaysKeepData(boolean alwaysKeepData) {
83+
this.alwaysKeepData = alwaysKeepData;
84+
}
85+
86+
@Override
87+
public void save(BehaivStorage storage) throws IOException {
88+
final File dataFile = storage.getDataFile(id);
89+
final PredictionSet predictionSet = DataMappingUtils.createPredictionSet(data);
90+
try(FileOutputStream fileOutputStream = new FileOutputStream(dataFile)) {
91+
predictionSet.writeTo(fileOutputStream);
92+
}
93+
}
94+
95+
@Override
96+
public void restore(BehaivStorage storage) throws IOException {
97+
final File dataFile = storage.getDataFile(id);
98+
try (FileInputStream fileInputStream = new FileInputStream(dataFile)) {
99+
final PredictionSet predictionSet = PredictionSet.parseFrom(fileInputStream);
100+
data = DataMappingUtils.dataFromPredictionSet(predictionSet);
101+
}
102+
}
103+
}

‎src/main/java/de/dmi3y/behaiv/kernel/Kernel.java

Lines changed: 15 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,37 @@
11
package de.dmi3y.behaiv.kernel;
22

3-
import com.fasterxml.jackson.core.type.TypeReference;
4-
import com.fasterxml.jackson.databind.ObjectMapper;
53
import de.dmi3y.behaiv.storage.BehaivStorage;
64
import de.dmi3y.behaiv.tools.Pair;
75

8-
import java.io.BufferedReader;
9-
import java.io.BufferedWriter;
10-
import java.io.FileReader;
11-
import java.io.FileWriter;
126
import java.io.IOException;
13-
import java.util.ArrayList;
147
import java.util.List;
158

16-
public abstract class Kernel {
9+
public interface Kernel {
10+
void setId(String id);
1711

18-
protected String id;
19-
protected Long treshold = 10L;
20-
protected ObjectMapper objectMapper;
21-
protected boolean partialFitAllowed = false;
22-
protected boolean alwaysKeepData = true;
12+
boolean isEmpty();
2313

24-
public Kernel(String id) {
25-
this.id = id;
26-
objectMapper = new ObjectMapper();
27-
}
14+
void fit(List<Pair<List<Double>, String>> data);
2815

16+
void fit();
2917

30-
public void setId(String id) {
31-
this.id = id;
32-
}
18+
void setTreshold(Long treshold);
3319

20+
boolean readyToPredict();
3421

35-
//list<features>, label
36-
protected List<Pair<List<Double>, String>> data = new ArrayList<>();
22+
void update(List<Pair<List<Double>, String>> data);
3723

24+
boolean isPartialFitAllowed();
3825

39-
public abstract boolean isEmpty();
26+
void updateSingle(List<Double> features, String label);
4027

41-
public abstract void fit(List<Pair<List<Double>, String>> data);
28+
String predictOne(List<Double> features);
4229

43-
public void fit() {
44-
fit(this.data);
45-
}
30+
boolean isAlwaysKeepData();
4631

47-
public void setTreshold(Long treshold) {
48-
this.treshold = treshold;
49-
}
32+
void setAlwaysKeepData(boolean alwaysKeepData);
5033

51-
public boolean readyToPredict() {
52-
return data.size() > treshold;
53-
}
34+
void save(BehaivStorage storage) throws IOException;
5435

55-
public void update(List<Pair<List<Double>, String>> data) {
56-
}
57-
58-
public boolean isPartialFitAllowed() {
59-
return partialFitAllowed;
60-
}
61-
62-
public void updateSingle(List<Double> features, String label) {
63-
data.add(new Pair<>(features, label));
64-
}
65-
66-
public abstract String predictOne(List<Double> features);
67-
68-
public boolean isAlwaysKeepData() {
69-
return alwaysKeepData;
70-
}
71-
72-
public void setAlwaysKeepData(boolean alwaysKeepData) {
73-
this.alwaysKeepData = alwaysKeepData;
74-
}
75-
76-
public void save(BehaivStorage storage) throws IOException {
77-
78-
try (final BufferedWriter writer = new BufferedWriter(new FileWriter(storage.getDataFile(id)))) {
79-
writer.write(objectMapper.writeValueAsString(data));
80-
}
81-
}
82-
83-
public void restore(BehaivStorage storage) throws IOException {
84-
final TypeReference<List<Pair<List<Double>, String>>> typeReference = new TypeReference<List<Pair<List<Double>, String>>>() {
85-
};
86-
try (final BufferedReader reader = new BufferedReader(new FileReader(storage.getDataFile(id)))) {
87-
final String content = reader.readLine();
88-
if (content == null || content.isEmpty()) {
89-
data = new ArrayList<>();
90-
} else {
91-
data = objectMapper.readValue(content, typeReference);
92-
}
93-
}
94-
}
36+
void restore(BehaivStorage storage) throws IOException;
9537
}

‎src/main/java/de/dmi3y/behaiv/kernel/LogisticRegressionKernel.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import static de.dmi3y.behaiv.tools.DataMappingUtils.toDistinctListOfPairValues;
1818
import static de.dmi3y.behaiv.tools.DataMappingUtils.toInput2dArray;
1919

20-
public class LogisticRegressionKernel extends Kernel {
20+
public class LogisticRegressionKernel extends BaseKernel {
2121

2222
protected List<String> labels = new ArrayList<>();
2323
private Random rand;

‎src/main/java/de/dmi3y/behaiv/tools/DataMappingUtils.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package de.dmi3y.behaiv.tools;
22

33
import org.apache.commons.lang3.ArrayUtils;
4+
import tech.donau.behaiv.proto.Data;
5+
import tech.donau.behaiv.proto.Prediction;
6+
import tech.donau.behaiv.proto.PredictionSet;
47

58
import java.util.ArrayList;
69
import java.util.HashSet;
@@ -14,6 +17,38 @@ private DataMappingUtils() {
1417
// Unused utility class
1518
}
1619

20+
public static PredictionSet createPredictionSet(List<Pair<List<Double>, String>> data) {
21+
final PredictionSet.Builder predictionSetBuilder = PredictionSet.newBuilder();
22+
for (int i = 0; i < data.size(); i++) {
23+
final List<Double> weights = data.get(i).getKey();
24+
final ArrayList<Data> row = new ArrayList<>();
25+
for (int j = 0; j < weights.size(); j++) {
26+
row.add(Data.newBuilder().setKey("key"+j).setValue(weights.get(j)).build());
27+
}
28+
predictionSetBuilder.addPrediction(
29+
Prediction.newBuilder()
30+
.addAllData(row)
31+
.setLabel(data.get(i).getValue())
32+
.build()
33+
);
34+
}
35+
return predictionSetBuilder.build();
36+
}
37+
38+
public static List<Pair<List<Double>, String>> dataFromPredictionSet(PredictionSet predictionSet) {
39+
final ArrayList<Pair<List<Double>, String>> data = new ArrayList<>();
40+
for (int i = 0; i < predictionSet.getPredictionCount(); i++) {
41+
final Prediction prediction = predictionSet.getPrediction(i);
42+
final List<Data> dataList = prediction.getDataList();
43+
final String label = prediction.getLabel();
44+
final List<Double> entries = new ArrayList<>();
45+
for (Data entry : dataList) {
46+
entries.add(entry.getValue());
47+
}
48+
data.add(Pair.create(entries, label));
49+
}
50+
return data;
51+
}
1752

1853
public static List<String> toDistinctListOfPairValues(List<Pair<List<Double>, String>> data) {
1954
Set<String> setOfValues = new HashSet<>();

0 commit comments

Comments
 (0)