--- /dev/null
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>hu.user</groupId>
+ <artifactId>CreditCardFraudDetect</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>CommonDto</artifactId>
+
+ <properties>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.14.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jdk8</artifactId>
+ <version>2.14.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.24</version>
+ <optional>true</optional>
+ </dependency>
+
+
+
+ </dependencies>
+
+</project>
\ No newline at end of file
--- /dev/null
+FROM adoptopenjdk:14-jre-hotspot
+WORKDIR /opt/app
+COPY target/*.jar core.jar
+CMD ["java","-jar","core.jar"]
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.6.1</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>hu.user</groupId>
+ <artifactId>Core</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>Core</name>
+ <description>Core</description>
+ <properties>
+ <java.version>11</java.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web-services</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-jdbc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>wsdl4j</groupId>
+ <artifactId>wsdl4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-basics</artifactId>
+ <version>0.9.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-value-constructor</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>hu.user</groupId>
+ <artifactId>CommonDto</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>jaxb2-maven-plugin</artifactId>
+ <version>2.5.0</version>
+ <executions>
+ <execution>
+ <id>xjc</id>
+ <goals>
+ <goal>xjc</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>-Xvalue-constructor</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <sources>
+ <source>${project.basedir}/src/main/resources/detection.xsd</source>
+ </sources>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+</project>
--- /dev/null
+package hu.user.core;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class CoreApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(CoreApplication.class, args);
+ }
+
+}
--- /dev/null
+package hu.user.core.config;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+public class RestConfig {
+
+ @Bean
+ public RestTemplate getRestTemplate(RestTemplateBuilder builder) {
+ RestTemplate restTemplate = builder
+ .build();
+ return restTemplate;
+
+ }
+}
--- /dev/null
+package hu.user.core.config;
+
+import org.springframework.stereotype.Component;
+
+import hu.user.core.dto.StatisticsDto;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Component
+@Getter
+@Setter
+public class StatisticConfig {
+
+ private String cardNumberFieldName;
+ private String transactionTimeFieldName;
+ private boolean enabled;
+
+ public void setParameters(StatisticsDto statisticsDto) {
+ this.cardNumberFieldName = statisticsDto.getCardNumberFieldName();
+ this.transactionTimeFieldName = statisticsDto.getTransactionTimeFieldName();
+ this.enabled = enabled;
+ }
+
+ public void deleteParameters() {
+ this.cardNumberFieldName = null;
+ this.transactionTimeFieldName = null;
+ this.enabled = false;
+ }
+
+}
--- /dev/null
+package hu.user.core.config;
+
+import org.springframework.boot.web.servlet.ServletRegistrationBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.ws.config.annotation.EnableWs;
+import org.springframework.ws.config.annotation.WsConfigurerAdapter;
+import org.springframework.ws.transport.http.MessageDispatcherServlet;
+import org.springframework.ws.wsdl.wsdl11.DefaultWsdl11Definition;
+import org.springframework.xml.xsd.SimpleXsdSchema;
+import org.springframework.xml.xsd.XsdSchema;
+
+@EnableWs
+@Configuration
+public class WebServiceConfig extends WsConfigurerAdapter {
+ @Bean
+ public ServletRegistrationBean<MessageDispatcherServlet> messageDispatcherServlet(ApplicationContext applicationContext) {
+ MessageDispatcherServlet servlet = new MessageDispatcherServlet();
+ servlet.setApplicationContext(applicationContext);
+ servlet.setTransformWsdlLocations(true);
+ return new ServletRegistrationBean<>(servlet, "/ws/*");
+ }
+
+ @Bean(name = "detection")
+ public DefaultWsdl11Definition defaultWsdl11Definition(XsdSchema countriesSchema) {
+ DefaultWsdl11Definition wsdl11Definition = new DefaultWsdl11Definition();
+ wsdl11Definition.setPortTypeName("DetectionPort");
+ wsdl11Definition.setLocationUri("/ws");
+ wsdl11Definition.setTargetNamespace("http://spring.io/guides/gs-producing-web-service");
+ wsdl11Definition.setSchema(countriesSchema);
+ return wsdl11Definition;
+ }
+
+ @Bean
+ public XsdSchema countriesSchema() {
+ return new SimpleXsdSchema(new ClassPathResource("detection.xsd"));
+ }
+}
--- /dev/null
+package hu.user.core.controller;
+
+import java.util.List;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.core.estimators.EstimatorContainer;
+
+@Controller
+public class EstimatorController {
+
+ private EstimatorContainer estimatorContainer;
+
+ public EstimatorController(EstimatorContainer estimatorContainer) {
+ this.estimatorContainer = estimatorContainer;
+ }
+
+ @PostMapping("/estimator")
+ public ResponseEntity<List<EstimatorRegistryParameterDto>> addEstimator(
+ @RequestBody EstimatorRegistryParameterDto estimatorRegistryParameterDto) {
+ List<EstimatorRegistryParameterDto> registeredEstimators =
+ estimatorContainer.addEstimator(estimatorRegistryParameterDto);
+ return new ResponseEntity<>(registeredEstimators, HttpStatus.OK);
+ }
+
+ @GetMapping("/estimator")
+ public ResponseEntity<List<EstimatorRegistryParameterDto>> getEstimators() {
+ return new ResponseEntity<>(estimatorContainer.getEstimator(), HttpStatus.OK);
+ }
+
+ @PutMapping("estimator")
+ public ResponseEntity<List<EstimatorRegistryParameterDto>> modifyEstimator(
+ @RequestBody EstimatorRegistryParameterDto estimatorRegistryParameterDto) {
+ return new ResponseEntity<>(estimatorContainer.updateEstimator(estimatorRegistryParameterDto), HttpStatus.OK);
+ }
+
+ @DeleteMapping("estimator/{id}")
+ public ResponseEntity<List<EstimatorRegistryParameterDto>> deleteEstimator(@PathVariable("id") int id) {
+ return new ResponseEntity<>(estimatorContainer.deleteEstimator(id), HttpStatus.OK);
+ }
+}
--- /dev/null
+package hu.user.core.controller;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import hu.user.common.dto.ConfusionMatrixDto;
+import hu.user.core.dto.ObservedValue;
+import hu.user.core.service.FraudService;
+import hu.user.core.service.StatisticService;
+
+@Controller
+public class StatisticController {
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Autowired private FraudService fraudService;
+ @Autowired private StatisticService statisticService;
+
+ @GetMapping("/confusion_matrix")
+ public ResponseEntity<ConfusionMatrixDto> getConfusionMatrix() {
+ logger.info("get confusion matrix");
+ return new ResponseEntity<>(fraudService.qualify(), HttpStatus.OK);
+ }
+
+ @PostMapping("/observed")
+ public ResponseEntity<?> storeObservedValues(@RequestBody List<ObservedValue> observedValues) {
+ statisticService.storeObservedValues(observedValues);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
+
+ @GetMapping("/transaction")
+ public ResponseEntity<Integer> getAllTransaction() {
+ return new ResponseEntity<>(statisticService.getAllTransaction(), HttpStatus.OK);
+ }
+
+ @GetMapping("/observed")
+ public ResponseEntity<Integer> getAllObservedTransaction() {
+ return new ResponseEntity<>(statisticService.getAllObservedTransaction(), HttpStatus.OK);
+ }
+
+ @PostMapping("/transaction_clear")
+ public ResponseEntity<?> deleteAllTransaction() {
+ fraudService.deleteAllFraud();
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
+ }
+
+
--- /dev/null
+package hu.user.core.domain;
+
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Entity
+@Getter
+@Setter
+public class Fraud {
+
+ @Id
+// @GeneratedValue(strategy = GenerationType.AUTO)
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
+ private Long Id;
+ private String transactionUuid;
+ private Integer predictedValue;
+
+ private Integer observedValue;
+
+ private boolean predicted;
+
+ private boolean observed;
+
+ private double positiveProbability;
+
+ private double negativeProbability;
+
+ public Fraud(String transactionUuid, int predictedValue,double positiveProbability,double negativeProbability) {
+ this.transactionUuid = transactionUuid;
+ this.predictedValue = predictedValue;
+ this.predicted = true;
+ this.observed = false;
+ this.positiveProbability = positiveProbability;
+ this.negativeProbability = negativeProbability;
+ }
+
+ public Fraud() {
+ }
+}
--- /dev/null
+package hu.user.core.dto;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@AllArgsConstructor
+@NoArgsConstructor
+@Getter
+@Setter
+public class CommulatedResponseFromEstimator implements Serializable {
+
+ private Map<Integer, ResponseFromSingleEstimator> response = new HashMap<>();
+
+}
--- /dev/null
+package hu.user.core.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class ObservedValue {
+
+ private String uuid;
+ private int observedFraud;
+}
--- /dev/null
+package hu.user.core.dto;
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Getter
+@Setter
+public class ResponseFromSingleEstimator implements Serializable {
+
+ protected int prediction;
+ protected double negativeProbability;
+ protected double positiveProbability;
+
+ public ResponseFromSingleEstimator(@JsonProperty(value = "prediction", required = true) int prediction,
+ @JsonProperty(value = "negative_probability", required = true) double negativeProbability,
+ @JsonProperty(value = "positive_probability", required = true) double positiveProbability) {
+ this.prediction = prediction;
+ this.negativeProbability = negativeProbability;
+ this.positiveProbability = positiveProbability;
+ }
+}
--- /dev/null
+package hu.user.core.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+
+@Getter
+public class StatisticsDto {
+
+ private String cardNumberFieldName;
+ private String transactionTimeFieldName;
+ private boolean enabled;
+
+ public StatisticsDto(@JsonProperty(value = "card_number_field_name", required = true) String cardNumberFieldName,
+ @JsonProperty(value = "transaction_time_field_name", required = true) String transactionTimeFieldName,
+ @JsonProperty(value = "enabled", required = true) boolean enabled) {
+
+ this.cardNumberFieldName = cardNumberFieldName;
+ this.transactionTimeFieldName = transactionTimeFieldName;
+ this.enabled = enabled;
+ }
+}
--- /dev/null
+package hu.user.core.endpoint;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.ws.server.endpoint.annotation.Endpoint;
+import org.springframework.ws.server.endpoint.annotation.PayloadRoot;
+import org.springframework.ws.server.endpoint.annotation.RequestPayload;
+import org.springframework.ws.server.endpoint.annotation.ResponsePayload;
+
+import hu.user.core.service.StatisticService;
+import hu.user.core.service.VoteModul;
+import io.spring.guides.gs_producing_web_service.DetectionResponse;
+import io.spring.guides.gs_producing_web_service.GenerateDetectionRequest;
+import io.spring.guides.gs_producing_web_service.GenerateGenericDetectionRequest;
+import io.spring.guides.gs_producing_web_service.ObjectFactory;
+import io.spring.guides.gs_producing_web_service.GenerateDetectionResponse;
+import io.spring.guides.gs_producing_web_service.StoreRealTransactionRequest;
+import io.spring.guides.gs_producing_web_service.StoreRealTransactionResponse;
+import io.spring.guides.gs_producing_web_service.StoreResult;
+import lombok.AllArgsConstructor;
+
+@Endpoint
+@AllArgsConstructor
+public class WebserviceEndpoint {
+
+ private static final String NAMESPACE_URI = "http://spring.io/guides/gs-producing-web-service";
+
+ private VoteModul voteModul;
+ private StatisticService statisticService;
+
+
+
+ @PayloadRoot(namespace = NAMESPACE_URI, localPart = "generateDetectionRequest")
+ @ResponsePayload
+ public GenerateDetectionResponse getPrediction(@RequestPayload GenerateDetectionRequest generateDetectionRequest) {
+ ObjectFactory objectFactory =new ObjectFactory();
+ DetectionResponse detectionResponse = voteModul.voting(generateDetectionRequest);
+ GenerateDetectionResponse generateDetectionResponse = objectFactory.createGenerateDetectionResponse();
+ generateDetectionResponse.setDetectionResponse(detectionResponse);
+ return generateDetectionResponse;
+ }
+
+ @PayloadRoot(namespace = NAMESPACE_URI, localPart = "generateGenericDetectionRequest")
+ @ResponsePayload
+ public GenerateDetectionResponse getGenericPrediction(@RequestPayload
+ GenerateGenericDetectionRequest generateGenericDetectionRequest) {
+ ObjectFactory objectFactory =new ObjectFactory();
+ DetectionResponse detectionResponse = voteModul.genericVoting(generateGenericDetectionRequest);
+ GenerateDetectionResponse generateDetectionResponse = objectFactory.createGenerateDetectionResponse();
+ generateDetectionResponse.setDetectionResponse(detectionResponse);
+ return generateDetectionResponse;
+ }
+
+ @PayloadRoot(namespace = NAMESPACE_URI, localPart = "storeRealTransactionRequest")
+ @ResponsePayload
+ public StoreRealTransactionResponse storeRealTransactions(
+ @RequestPayload StoreRealTransactionRequest storeRealTransactionRequest) {
+ ObjectFactory objectFactory =new ObjectFactory();
+ StoreRealTransactionResponse storeRealTransactionResponse=objectFactory.createStoreRealTransactionResponse();
+ StoreResult storeResult = statisticService.storeRealTransactions(storeRealTransactionRequest);
+ storeRealTransactionResponse.setStoreResult(storeResult);
+ return storeRealTransactionResponse;
+
+ }
+
+}
--- /dev/null
+package hu.user.core.estimators;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Component;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.core.service.TrainClient;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Component
+@Getter
+@Setter
+@Slf4j
+public class EstimatorContainer {
+
+ private static final String ID = "id";
+
+ private TrainClient trainClient;
+ private Map<Integer, Integer> estimatorWeightById = new HashMap<>();
+ private String transactionIdentifierFieldName;
+
+ public EstimatorContainer(TrainClient trainClient) {
+ this.trainClient = trainClient;
+ }
+
+ public List<EstimatorRegistryParameterDto> addEstimator(EstimatorRegistryParameterDto registryParameterDto) {
+ if (transactionIdentifierFieldName == null) {
+ transactionIdentifierFieldName = trainClient.getTransactionIdentifierFieldNameByEstimatorId(
+ Map.of(ID, String.valueOf(registryParameterDto.getEstimatorId())));
+ }
+ estimatorWeightById.put(registryParameterDto.getEstimatorId(), registryParameterDto.getWeight());
+ log.debug(
+ "Estimator registered, estimator id: " + registryParameterDto.getEstimatorId() + ", weight: " + registryParameterDto.getWeight());
+ return convertToDtos();
+ }
+
+ public List<EstimatorRegistryParameterDto> getEstimator() {
+ return convertToDtos();
+ }
+
+ public List<EstimatorRegistryParameterDto> updateEstimator(EstimatorRegistryParameterDto registryParameterDto) {
+ int id = registryParameterDto.getEstimatorId();
+ if (estimatorWeightById.containsKey(id)) {
+ estimatorWeightById.put(registryParameterDto.getEstimatorId(), registryParameterDto.getWeight());
+ log.debug(
+ "Estimator weight updated, estimator id: " + registryParameterDto.getEstimatorId() + ", weight: " + registryParameterDto.getWeight());
+ }
+ return convertToDtos();
+ }
+
+ public List<EstimatorRegistryParameterDto> deleteEstimator(int estimatorId) {
+ if (estimatorWeightById.containsKey(estimatorId)) {
+ estimatorWeightById.remove(estimatorId);
+ log.debug("Estimator removed, estimator id: " + estimatorId);
+ }
+ return convertToDtos();
+ }
+
+ private List<EstimatorRegistryParameterDto> convertToDtos() {
+ return estimatorWeightById.entrySet().stream()
+ .map(e -> new EstimatorRegistryParameterDto(e.getKey(), e.getValue())).collect(Collectors.toList());
+ }
+}
--- /dev/null
+package hu.user.core.estimators;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class WeightAndTransactionIdentifierFieldName {
+
+ private int weight;
+ private String transactionIdentifierFieldName;
+
+}
--- /dev/null
+package hu.user.core.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+
+import hu.user.core.domain.Fraud;
+
+public interface FraudRepository extends JpaRepository<Fraud,Long> {
+
+
+ @Query("SELECT COUNT(f) FROM Fraud f WHERE f.predicted = true and f.observed=true and f.predictedValue=1 and f.observedValue=1")
+ int getNumberOfTruePositivePrediction();
+ @Query("SELECT COUNT(f) FROM Fraud f WHERE f.predicted = true and f.observed=true and f.predictedValue=1 and f.observedValue=0")
+ int getNumberOfFalsePositivePrediction();
+ @Query("SELECT COUNT(f) FROM Fraud f WHERE f.predicted = true and f.observed=true and f.predictedValue=0 and f.observedValue=0")
+ int getNumberOfTrueNegativePrediction();
+ @Query("SELECT COUNT(f) FROM Fraud f WHERE f.predicted = true and f.observed=true and f.predictedValue=0 and f.observedValue=1")
+ int getNumberOfFalseNegativePrediction();
+ public Fraud getFraudByTransactionUuid(String transactionUuid);
+ public int countAllByObserved(boolean observed);
+ @Query("SELECT COUNT(f) FROM Fraud f")
+ public int countAllFraud();
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import hu.user.common.dto.ConfusionMatrixDto;
+import hu.user.core.domain.Fraud;
+import hu.user.core.dto.ObservedValue;
+import hu.user.core.repository.FraudRepository;
+
+@Service
+public class FraudService {
+
+ @Autowired private FraudRepository fraudRepository;
+
+ public void savePredictionResult(String transactionUuid, int predictedValue, double positiveProbability,double negativeProbability) {
+ Fraud fraud = fraudRepository.getFraudByTransactionUuid(transactionUuid);
+ if (fraud != null) {
+ fraudRepository.delete(fraud);
+ }
+ fraud = new Fraud(transactionUuid, predictedValue, positiveProbability, negativeProbability);
+ fraudRepository.save(fraud);
+ }
+
+ public ConfusionMatrixDto qualify() {
+ int truePositivePredictionNumber = fraudRepository.getNumberOfTruePositivePrediction();
+ int falsePositivePredictionNumber = fraudRepository.getNumberOfFalsePositivePrediction();
+ int trueNegativePredictionNumber = fraudRepository.getNumberOfTrueNegativePrediction();
+ int falseNegativePredictionNumber = fraudRepository.getNumberOfFalseNegativePrediction();
+ ConfusionMatrixDto confusionMatrixDto =
+ new ConfusionMatrixDto(truePositivePredictionNumber, falsePositivePredictionNumber,
+ trueNegativePredictionNumber, falseNegativePredictionNumber);
+ return confusionMatrixDto;
+ }
+
+ public void deleteAllFraud() {
+ fraudRepository.deleteAll();
+ }
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.core.dto.ResponseFromSingleEstimator;
+import io.spring.guides.gs_producing_web_service.DetectionRequest;
+import io.spring.guides.gs_producing_web_service.GenericDetectionRequest;
+
+@Service
+public class PredictionService {
+
+ @Value("${estimatorUrl}/prediction") URI estimatorUrl;
+
+ @Autowired private RestTemplate restTemplate;
+
+ public Map<Integer, ResponseFromSingleEstimator> getPrediction(DetectionRequest detectionRequest) {
+ Map<String, Object> requestParameters = new HashMap<>();
+ requestParameters.put("card_number", detectionRequest.getCardNumber());
+ requestParameters.put("transaction_type", detectionRequest.getTransactionType());
+ requestParameters.put("timestamp", detectionRequest.getTimestamp());
+ requestParameters.put("amount", detectionRequest.getAmount());
+ requestParameters.put("currency_name", detectionRequest.getCurrencyName());
+ requestParameters.put("response_code", detectionRequest.getResponseCode());
+ requestParameters.put("country_name", detectionRequest.getCountryName());
+ requestParameters.put("vendor_code", detectionRequest.getVendorCode());
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<Map<String, Object>> httpEntity = new HttpEntity<>(requestParameters, httpHeaders);
+ ParameterizedTypeReference<Map<Integer, ResponseFromSingleEstimator>> typeReference =
+ new ParameterizedTypeReference<>() {
+ };
+ ResponseEntity<Map<Integer,ResponseFromSingleEstimator>> responseEntity =
+ restTemplate.exchange(estimatorUrl, HttpMethod.POST,httpEntity, typeReference);
+ Map<Integer,ResponseFromSingleEstimator> response= responseEntity.getBody();
+ return response;
+ }
+
+ public Map<Integer, ResponseFromSingleEstimator> getGenericPrediction(List<GenericDetectionRequest> requests) {
+ Map<String, Object> requestParameters = new HashMap<>();
+ requests.stream().forEach(e->requestParameters.put(e.getFieldName(),e.getValue()));
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<Map<String, Object>> httpEntity = new HttpEntity<>(requestParameters, httpHeaders);
+ ParameterizedTypeReference<Map<Integer, ResponseFromSingleEstimator>> typeReference =
+ new ParameterizedTypeReference<>() {
+ };
+ ResponseEntity<Map<Integer,ResponseFromSingleEstimator>> responseEntity =
+ restTemplate.exchange(estimatorUrl, HttpMethod.POST,httpEntity, typeReference);
+ Map<Integer,ResponseFromSingleEstimator> response= responseEntity.getBody();
+ return response;
+ }
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import hu.user.core.domain.Fraud;
+import hu.user.core.dto.ObservedValue;
+import hu.user.core.repository.FraudRepository;
+import io.spring.guides.gs_producing_web_service.StoreRealTransactionRequest;
+import io.spring.guides.gs_producing_web_service.StoreRequest;
+import io.spring.guides.gs_producing_web_service.StoreResult;
+import lombok.AllArgsConstructor;
+
+@Service
+@AllArgsConstructor
+public class StatisticService {
+
+ private final static String SUCCESSFUL = "successful";
+ private FraudRepository fraudRepository;
+
+ public StoreResult storeRealTransactions(StoreRealTransactionRequest storeRealTransactionRequest) {
+ int recordNumber;
+ List<Fraud> observedFrauds = new ArrayList<>();
+ List<StoreRequest> requests = storeRealTransactionRequest.getStoreRequest();
+ for (StoreRequest request : requests) {
+ String transactionUuid = request.getTransactionId();
+ Fraud fraud = fraudRepository.getFraudByTransactionUuid(transactionUuid);
+ fraud.setObserved(true);
+ fraud.setObservedValue(request.getFraud());
+ observedFrauds.add(fraud);
+ }
+ fraudRepository.saveAll(observedFrauds);
+ StoreResult result = new StoreResult(SUCCESSFUL, observedFrauds.size());
+ return result;
+ }
+
+ public void storeObservedValues(List<ObservedValue> observedValues) {
+ List<Fraud> observedFrauds = new ArrayList<>();
+ for (ObservedValue observedValue : observedValues) {
+ String transactionUuid = observedValue.getUuid();
+ Fraud fraud = fraudRepository.getFraudByTransactionUuid(transactionUuid);
+ if (fraud != null) {
+ fraud.setObserved(true);
+ fraud.setObservedValue(observedValue.getObservedFraud());
+ observedFrauds.add(fraud);
+ }
+ }
+ if (!observedFrauds.isEmpty()) {
+ fraudRepository.saveAll(observedFrauds);
+ }
+ }
+
+ public int getAllTransaction() {
+ return fraudRepository.countAllFraud();
+ }
+
+ public int getAllObservedTransaction() {
+ return fraudRepository.countAllByObserved(true);
+ }
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.annotation.ScopedProxyMode;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+@Service
+@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS)
+public class TrainClient {
+
+ private static final String TRANSACTION_IDENTIFIER_FIELD_NAME_PATH = "/transaction_identifier_field_name";
+ @Value("${trainUrl}") String trainUrl;
+ @Autowired private RestTemplate restTemplate;
+
+ public String getTransactionIdentifierFieldNameByEstimatorId(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TRANSACTION_IDENTIFIER_FIELD_NAME_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> responseFromTrainModule = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = responseFromTrainModule.getBody();
+ return result;
+ }
+
+
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.web.util.UriComponentsBuilder;
+
+public class UrlBuilder {
+
+ private String url;
+ private Map<String, String> queryParams = new HashMap<>();
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Map<String, String> getQueryParams() {
+ return queryParams;
+ }
+
+ public void build(String url, Map<String, String> queryValueByParamName) {
+ List<String> paramNames = new ArrayList<>();
+ List<String> paramValues = new ArrayList<>();
+ for (var entry : queryValueByParamName.entrySet()) {
+ paramNames.add(entry.getKey());
+ paramValues.add(entry.getValue());
+ }
+ buildUrl(url, paramNames);
+ fillQueryParams(paramNames, paramValues);
+ }
+
+ private void buildUrl(String url, List<String> queryParamNames) {
+ String urlTemplate;
+ UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromHttpUrl(url);
+ for (String queryParamName : queryParamNames) {
+ uriComponentsBuilder = uriComponentsBuilder.queryParam(queryParamName, "{" + queryParamName + "}");
+ }
+ this.url = uriComponentsBuilder.encode().toUriString();
+ }
+
+ private void fillQueryParams(List<String> paramNames, List<String> paramValuee) {
+ Map<String, String> params = new HashMap<>();
+ for (int i = 0; i < paramNames.size(); i++) {
+ params.put(paramNames.get(i), paramValuee.get(i));
+ }
+ this.queryParams = params;
+ }
+}
--- /dev/null
+package hu.user.core.service;
+
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import hu.user.core.dto.ResponseFromSingleEstimator;
+import hu.user.core.estimators.EstimatorContainer;
+import io.spring.guides.gs_producing_web_service.DetectionRequest;
+import io.spring.guides.gs_producing_web_service.DetectionResponse;
+import io.spring.guides.gs_producing_web_service.GenerateDetectionRequest;
+import io.spring.guides.gs_producing_web_service.GenerateGenericDetectionRequest;
+import io.spring.guides.gs_producing_web_service.GenericDetectionRequest;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Getter
+@Setter
+@Slf4j
+public class VoteModul {
+
+ private EstimatorContainer estimatorContainer;
+ private PredictionService predictionService;
+ private FraudService fraudService;
+
+ public VoteModul(EstimatorContainer estimatorContainer, PredictionService predictionService,
+ FraudService fraudService) {
+ this.estimatorContainer = estimatorContainer;
+ this.predictionService = predictionService;
+ this.fraudService = fraudService;
+ }
+
+ public DetectionResponse voting(GenerateDetectionRequest generateDetectionRequest) {
+ DetectionRequest detectionRequest = generateDetectionRequest.getDetectionRequest();
+ Map<Integer, ResponseFromSingleEstimator> responseFromEstimator =
+ predictionService.getPrediction(detectionRequest);
+ DetectionResponse detectionResponse = evaluate(responseFromEstimator);
+ String transactionId = generateDetectionRequest.getTransactionId();
+ fraudService.savePredictionResult(transactionId, detectionResponse.getPrediction(),
+ detectionResponse.getPositiveProbability(), detectionResponse.getNegativeProbability());
+ log.info(
+ "Prediction: " + detectionResponse.getPrediction() + ", positive probability: " + detectionResponse.getPositiveProbability() + ", negative probability: " + detectionResponse.getNegativeProbability());
+ return detectionResponse;
+ }
+
+ public DetectionResponse genericVoting(GenerateGenericDetectionRequest generateGenericDetectionRequest) {
+ List<GenericDetectionRequest> genericDetectionRequests = generateGenericDetectionRequest.getDetectionRequest();
+ Map<Integer, ResponseFromSingleEstimator> responseFromEstimator =
+ predictionService.getGenericPrediction(genericDetectionRequests);
+ DetectionResponse detectionResponse = evaluate(responseFromEstimator);
+ String transactionIdentifierFieldName = estimatorContainer.getTransactionIdentifierFieldName();
+ String transactionIdentifierValue = null;
+ for (GenericDetectionRequest request : genericDetectionRequests) {
+ if (request.getFieldName().equals(transactionIdentifierFieldName)) {
+ transactionIdentifierValue = request.getValue();
+ }
+ }
+ fraudService.savePredictionResult(transactionIdentifierValue, detectionResponse.getPrediction(),
+ detectionResponse.getPositiveProbability(), detectionResponse.getNegativeProbability());
+ return detectionResponse;
+ }
+
+ private DetectionResponse evaluate(Map<Integer, ResponseFromSingleEstimator> responseFromEstimator) {
+ DetectionResponse summaryzedResponse = new DetectionResponse();
+ Map<Integer, Integer> estimatorWeightById = estimatorContainer.getEstimatorWeightById();
+ int noOKNumber = 0;
+ int oKNumber = 0;
+ double noOKProbability = 1;
+ double oKProbability = 1;
+
+ for (Map.Entry<Integer, ResponseFromSingleEstimator> entry : responseFromEstimator.entrySet()) {
+ int estimatorId = entry.getKey();
+ ResponseFromSingleEstimator responseFromSingleEstimator = entry.getValue();
+ int weight = estimatorWeightById.get(estimatorId);
+ if (responseFromSingleEstimator.getPrediction() == 1) {
+ noOKNumber += weight;
+ noOKProbability *= responseFromSingleEstimator.getPositiveProbability();
+ } else {
+ oKNumber += weight;
+ oKProbability *= responseFromSingleEstimator.getNegativeProbability();
+ }
+ }
+ if (oKNumber > noOKNumber) {
+ summaryzedResponse.setPrediction(0);
+ summaryzedResponse.setNegativeProbability(oKProbability);
+ summaryzedResponse.setPositiveProbability(1 - oKProbability);
+ } else if (oKNumber < noOKNumber) {
+ summaryzedResponse.setPrediction(1);
+ summaryzedResponse.setPositiveProbability(noOKProbability);
+ summaryzedResponse.setNegativeProbability(1 - noOKProbability);
+ }
+ return summaryzedResponse;
+ }
+}
--- /dev/null
+spring:
+ profiles:
+ default: "dev"
+ application:
+ name: Core
+ datasource:
+ url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/common_fraud?serverTimezone=UTC
+ username: root
+ password: pwd
+ jpa:
+ hibernate:
+ ddl-auto: create
+logging:
+ level:
+ root: info
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /home/tomi/log/${spring.application.name}.log
+server:
+ port: 8081
+estimatorUrl: http://localhost:8083
+trainUrl: http://localhost:8085
+showProgressRepeatTime: 500
+trainLivenessRepeatTime: 100
+
+---
+spring:
+ config:
+ activate:
+ on-profile: dockerized
+# jpa:
+# show-sql: true
+# generate-ddl: true
+# hibernate:
+# formar_sql: true
+server:
+ port: 8081
+logging:
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /opt/app/log/${spring.application.name}.log
+estimatorUrl: http://estimator:8083
+trainUrl: http://train:8085
+showProgressRepeatTime: 500
+trainLivenessRepeatTime: 100
+
+---
--- /dev/null
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:tns="http://spring.io/guides/gs-producing-web-service"
+ targetNamespace="http://spring.io/guides/gs-producing-web-service" elementFormDefault="qualified">
+
+ <xs:element name="generateDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="transaction_id" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="detection_request" type="tns:detection_request" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="generateGenericDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="detection_request" type="tns:generic_detection_request" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="storeRealTransactionRequest">
+ <xs:complexType>
+ <xs:sequence minOccurs="1" maxOccurs="unbounded">
+ <xs:element name="store_request" type="tns:store_request" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="storeRealTransactionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="store_result" type="tns:store_result" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+
+
+ <xs:element name="generateDetectionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="detection_response" type="tns:detection_response" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:complexType name="detection_request">
+ <xs:sequence>
+ <xs:element name="transaction_id" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="card_number" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="transaction_type" type="xs:int" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="timestamp" type="xs:dateTime" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="amount" type="xs:int" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="currency_name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="response_code" type="xs:int" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="country_name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="vendor_code" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="generic_detection_request">
+ <xs:sequence>
+ <xs:element name="field_name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="value" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="detection_response">
+ <xs:sequence>
+ <xs:element name="prediction" type="xs:int"/>
+ <xs:element name="negative_probability" type="xs:double"/>
+ <xs:element name="positive_probability" type="xs:double"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="store_request">
+ <xs:sequence>
+ <xs:element name="transaction_id" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="fraud" type="xs:int" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="store_result">
+ <xs:sequence>
+ <xs:element name="result" type="xs:string"/>
+ <xs:element name="record_number" type="xs:int"/>
+ </xs:sequence>
+ </xs:complexType>
+
+</xs:schema>
\ No newline at end of file
--- /dev/null
+package hu.user.core;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class CoreApplicationTests {
+
+ @Test
+ void contextLoads() {
+ }
+
+}
--- /dev/null
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
--- /dev/null
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# M2_HOME - location of maven2's installed home dir
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+ if [ -f /usr/local/etc/mavenrc ] ; then
+ . /usr/local/etc/mavenrc
+ fi
+
+ if [ -f /etc/mavenrc ] ; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ] ; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ MINGW*) mingw=true;;
+ Darwin*) darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ export JAVA_HOME="`/usr/libexec/java_home`"
+ else
+ export JAVA_HOME="/Library/Java/Home"
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=`java-config --jre-home`
+ fi
+fi
+
+if [ -z "$M2_HOME" ] ; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+
+ saveddir=`pwd`
+
+ M2_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ M2_HOME=`cd "$M2_HOME" && pwd`
+
+ cd "$saveddir"
+ # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --unix "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME="`(cd "$M2_HOME"; pwd)`"
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="`which javac`"
+ if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=`which readlink`
+ if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
+ if $darwin ; then
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
+ else
+ javaExecutable="`readlink -f \"$javaExecutable\"`"
+ fi
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaHome=`expr "$javaHome" : '\(.*\)/bin'`
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="`\\unset -f command; \\command -v java`"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+ if [ -z "$1" ]
+ then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ] ; do
+ if [ -d "$wdir"/.mvn ] ; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=`cd "$wdir/.."; pwd`
+ fi
+ # end of workaround
+ done
+ echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ echo "$(tr -s '\n' ' ' < "$1")"
+ fi
+}
+
+BASE_DIR=`find_maven_basedir "$(pwd)"`
+if [ -z "$BASE_DIR" ]; then
+ exit 1;
+fi
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found .mvn/wrapper/maven-wrapper.jar"
+ fi
+else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
+ fi
+ if [ -n "$MVNW_REPOURL" ]; then
+ jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ else
+ jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ fi
+ while IFS="=" read key value; do
+ case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
+ esac
+ done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Downloading from: $jarUrl"
+ fi
+ wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
+ if $cygwin; then
+ wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
+ fi
+
+ if command -v wget > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found wget ... using wget"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ else
+ wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ fi
+ elif command -v curl > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found curl ... using curl"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ curl -o "$wrapperJarPath" "$jarUrl" -f
+ else
+ curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
+ fi
+
+ else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Falling back to using Java to download"
+ fi
+ javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ # For Cygwin, switch paths to Windows format before running javac
+ if $cygwin; then
+ javaClass=`cygpath --path --windows "$javaClass"`
+ fi
+ if [ -e "$javaClass" ]; then
+ if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Compiling MavenWrapperDownloader.java ..."
+ fi
+ # Compiling the Java class
+ ("$JAVA_HOME/bin/javac" "$javaClass")
+ fi
+ if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ # Running the downloader
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Running MavenWrapperDownloader.java ..."
+ fi
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+if [ "$MVNW_VERBOSE" = true ]; then
+ echo $MAVEN_PROJECTBASEDIR
+fi
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --path --windows "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ $MAVEN_DEBUG_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.home=${M2_HOME}" \
+ "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
--- /dev/null
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM https://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
+if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+ IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Found %WRAPPER_JAR%
+ )
+) else (
+ if not "%MVNW_REPOURL%" == "" (
+ SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ )
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Couldn't find %WRAPPER_JAR%, downloading it ...
+ echo Downloading from: %DOWNLOAD_URL%
+ )
+
+ powershell -Command "&{"^
+ "$webclient = new-object System.Net.WebClient;"^
+ "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+ "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+ "}"^
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
+ "}"
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Finished downloading %WRAPPER_JAR%
+ )
+)
+@REM End of extension
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% ^
+ %JVM_CONFIG_MAVEN_PROPS% ^
+ %MAVEN_OPTS% ^
+ %MAVEN_DEBUG_OPTS% ^
+ -classpath %WRAPPER_JAR% ^
+ "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
+ %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
+if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%"=="on" pause
+
+if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
+
+cmd /C exit /B %ERROR_CODE%
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.5.3</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>hu.user</groupId>
+ <artifactId>CoreCli</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>CoreCli</name>
+ <description>CoreCli</description>
+ <properties>
+ <java.version>11</java.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web-services</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>4.0.1</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-core</artifactId>
+ <version>4.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>hu.user</groupId>
+ <artifactId>CommonDto</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>jaxb2-maven-plugin</artifactId>
+ <version>2.5.0</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-value-constructor</artifactId>
+ <version>3.0</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>xjc</id>
+ <goals>
+ <goal>xjc</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceType>wsdl</sourceType>
+ <sources>
+ <source>${project.basedir}/src/main/resources/wsdl/detection.wsdl</source>
+ </sources>
+ <arguments>
+ <argument>-Xvalue-constructor</argument>
+ </arguments>
+ <extension>true</extension>
+ <packageName>hu.user.corecli</packageName>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
--- /dev/null
+package hu.user.corecli;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.corecli.dto.ConfusionMatrix;
+import hu.user.corecli.service.CoreClient;
+import hu.user.corecli.service.EstimatorClient;
+
+@SpringBootApplication
+public class CoreCliApplication implements CommandLineRunner {
+
+ private final static String ADD_ESTIMATOR = "add_estimator";
+ private final static String GET_ESTIMATOR = "get_estimator";
+ private final static String UPDATE_ESTIMATOR_WEIGHT = "update_estimator_weight";
+ private final static String DELETE_ESTIMATOR = "delete_estimator";
+ private final static String PREDICT = "predict";
+ private final static String CONFUSION_MATRIX = "confusion_matrix";
+
+ private final static String ID = "id";
+ private final static String WEIGHT = "weight";
+
+ @Autowired EstimatorClient estimatorClient;
+ @Autowired CoreClient coreClient;
+
+ public static void main(String[] args) {
+ SpringApplication.run(CoreCliApplication.class, args);
+ }
+
+ @Override
+ public void run(String... args) throws Exception {
+ String idAsString;
+ String weigthAsString;
+ if (args.length > 0) {
+ String command = args[0];
+ switch (command) {
+ case ADD_ESTIMATOR:
+ if (args.length < 2) {
+ System.out.println("Estimator id is missing");
+ break;
+ } else {
+ idAsString = args[1];
+ if (!isParameterInteger(idAsString, ID)) {
+ break;
+ }
+ }
+ if (args.length < 3) {
+ System.out.println("Estimator's weight is missing");
+ break;
+ } else {
+ weigthAsString = args[2];
+ if (!isParameterInteger(weigthAsString, WEIGHT)) {
+ break;
+ }
+ }
+ List<Integer> registeredEstimatorIdsInEstimatorModul =
+ estimatorClient.addEstimatorInEstimatorModule(idAsString);
+ weigthAsString = args[2];
+ List<EstimatorRegistryParameterDto> registeredEstimatorIdAndWeigthInCoreModule =
+ coreClient.addEstimatorInCoreModule(idAsString, weigthAsString);
+ printResponseFromModules(registeredEstimatorIdsInEstimatorModul,
+ registeredEstimatorIdAndWeigthInCoreModule);
+ System.exit(0);
+ break;
+ case GET_ESTIMATOR:
+ registeredEstimatorIdsInEstimatorModul = estimatorClient.getEstimatorInEstimatorModule();
+ registeredEstimatorIdAndWeigthInCoreModule = coreClient.getEstimatorInCoreModule();
+ printResponseFromModules(registeredEstimatorIdsInEstimatorModul,
+ registeredEstimatorIdAndWeigthInCoreModule);
+ System.exit(0);
+ break;
+ case UPDATE_ESTIMATOR_WEIGHT:
+ if (args.length < 2) {
+ System.out.println("Estimator id is missing");
+ break;
+ } else {
+ idAsString = args[1];
+ if (!isParameterInteger(idAsString, ID)) {
+ break;
+ }
+ }
+ if (args.length < 3) {
+ System.out.println("Estimator's weight is missing");
+ break;
+ } else {
+ weigthAsString = args[2];
+ if (!isParameterInteger(weigthAsString, WEIGHT)) {
+ break;
+ }
+ }
+ registeredEstimatorIdsInEstimatorModul = estimatorClient.getEstimatorInEstimatorModule();
+ registeredEstimatorIdAndWeigthInCoreModule =
+ coreClient.modifyEstimatorWeightInCoreModule(idAsString, weigthAsString);
+ printResponseFromModules(registeredEstimatorIdsInEstimatorModul,
+ registeredEstimatorIdAndWeigthInCoreModule);
+ System.exit(0);
+ break;
+
+ case DELETE_ESTIMATOR:
+ if (args.length < 2) {
+ System.out.println("Estimator id is missing");
+ break;
+ }
+ idAsString = args[1];
+ if (!isParameterInteger(idAsString, ID)) {
+ break;
+ }
+ registeredEstimatorIdsInEstimatorModul = estimatorClient.deleteEstimatorInEstimatorModule(idAsString);
+ registeredEstimatorIdAndWeigthInCoreModule = coreClient.deleteEstimatorInCoreModule(idAsString);
+ printResponseFromModules(registeredEstimatorIdsInEstimatorModul,
+ registeredEstimatorIdAndWeigthInCoreModule);
+ System.exit(0);
+ break;
+ case CONFUSION_MATRIX:
+ ConfusionMatrix confusionMatrix = coreClient.getConfusionMatrix();
+ System.out.println("True positive: " + confusionMatrix.getTruePositive());
+ System.out.println("False positive: " + confusionMatrix.getFalsePositive());
+ System.out.println("True negative: " + confusionMatrix.getTrueNegative());
+ System.out.println("False negative: " + confusionMatrix.getFalseNegative());
+ System.exit(0);
+ default:
+ System.out.println("Available commmands:");
+ System.out.println(
+ "Register estimator in Estimator and Core module: add_estimator <estimator_id> <weight>");
+ System.out.println(
+ "List registered estimator in Estimator and Core module: get_estimator");
+ System.out.println(
+ "Update registered estimator's weight in Estimator and Core module: update_estimator <estimator_id> <weight>");
+ System.out.println(
+ "Delete registered estimator in Estimator and Core module: delete_estimator <estimator_id>");
+ System.out.println(
+ "Print system prediction quality: confusion_matrix");
+ System.exit(0);
+ }
+ }
+ }
+ private static boolean isParameterInteger(String parameter, String parameterName) {
+ boolean isInt = false;
+ try {
+ Integer.parseInt(parameter);
+ isInt = true;
+ } catch (NumberFormatException e) {
+ System.out.printf("The %s parameter isn't integer", parameterName);
+ }
+ return isInt;
+ }
+
+ private static void printResponseFromModules(List<Integer> registeredEstimatorIdsInEstimatorModul,
+ List<EstimatorRegistryParameterDto> registeredEstimatorIdAndWeigthInCoreModule) {
+ String ids = registeredEstimatorIdsInEstimatorModul.stream().map(e -> String.valueOf(e))
+ .collect(Collectors.joining(",", "{", "}"));
+ System.out.println("Registered estimator ids in Estimator module: " + ids);
+ String idsAndWeights = registeredEstimatorIdAndWeigthInCoreModule.stream()
+ .map(e -> "id: " + e.getEstimatorId() + " ; weight: " + e.getWeight())
+ .collect(Collectors.joining("; ", "{", "}"));
+ System.out.println("Registered estimator ids and weights in Core module: " + idsAndWeights);
+ }
+}
--- /dev/null
+package hu.user.corecli.config;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+public class RestConfig {
+
+ @Bean
+ public RestTemplate getRestTemplate(RestTemplateBuilder builder) {
+ RestTemplate restTemplate = builder
+ .build();
+ return restTemplate;
+
+ }
+}
--- /dev/null
+package hu.user.corecli.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class ConfusionMatrix {
+
+ private int truePositive;
+ private int falsePositive;
+ private int trueNegative;
+ private int falseNegative;
+
+ public ConfusionMatrix(@JsonProperty(value = "true_positive", required = true) int truePositive,
+ @JsonProperty(value = "false_positive", required = true) int falsePositive,
+ @JsonProperty(value = "true_negative", required = true) int trueNegative,
+ @JsonProperty(value = "false_negative", required = true) int falseNegative) {
+
+ this.truePositive = truePositive;
+ this.falsePositive = falsePositive;
+ this.trueNegative = trueNegative;
+ this.falseNegative = falseNegative;
+ }
+
+ public int getTruePositive() {
+ return truePositive;
+ }
+
+ public int getFalsePositive() {
+ return falsePositive;
+ }
+
+ public int getTrueNegative() {
+ return trueNegative;
+ }
+
+ public int getFalseNegative() {
+ return falseNegative;
+ }
+}
--- /dev/null
+package hu.user.corecli.service;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.corecli.dto.ConfusionMatrix;
+
+@Service
+public class CoreClient {
+
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String ESTIMATOR_DELETE_PATH = "/estimator/{id}";
+ private static final String CONFUSION_MATRIX_PATH = "/confusion_matrix";
+ private static final String WEIGHT = "weight";
+ private static final String ID = "id";
+ private static final String ESTIMATOR_ID = "estimator_id";
+
+ @Value("${coreUrl}") URI coreURI;
+
+ private RestTemplate restTemplate;
+
+ public CoreClient(RestTemplate restTemplate) {
+ this.restTemplate = restTemplate;
+ }
+
+ public List<EstimatorRegistryParameterDto> addEstimatorInCoreModule(String id, String weigth) {
+ Map<String, ?> input = Map.of(ESTIMATOR_ID, Integer.parseInt(id), WEIGHT, Integer.parseInt(weigth));
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> responseType = new ParameterizedTypeReference<>() {
+ };
+ List<EstimatorRegistryParameterDto> result =
+ exchangeAsMap(ESTIMATOR_PATH, HttpMethod.POST, input, responseType, Collections.emptyMap());
+ return result;
+ }
+
+ public List<EstimatorRegistryParameterDto> getEstimatorInCoreModule() {
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> responseType = new ParameterizedTypeReference<>() {
+ };
+ List<EstimatorRegistryParameterDto> result =
+ exchangeAsMap(ESTIMATOR_PATH, HttpMethod.GET, Collections.emptyMap(), responseType, Collections.emptyMap());
+ return result;
+ }
+
+ public List<EstimatorRegistryParameterDto> modifyEstimatorWeightInCoreModule(String id, String weigth) {
+ Map<String, ?> input = Map.of(ESTIMATOR_ID, Integer.parseInt(id), WEIGHT, Integer.parseInt(weigth));
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> responseType = new ParameterizedTypeReference<>() {
+ };
+ List<EstimatorRegistryParameterDto> result =
+ exchangeAsMap(ESTIMATOR_PATH, HttpMethod.PUT, input, responseType, Collections.emptyMap());
+ return result;
+ }
+ public List<EstimatorRegistryParameterDto> deleteEstimatorInCoreModule(String idAsString) {
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> responseType = new ParameterizedTypeReference<>() {
+ };
+ Map<String, ?> uriParam = Map.of(ID, Integer.parseInt(idAsString));
+ List<EstimatorRegistryParameterDto> result =
+ exchangeAsMap(ESTIMATOR_DELETE_PATH, HttpMethod.DELETE, Collections.emptyMap(), responseType, uriParam);
+ return result;
+ }
+ public ConfusionMatrix getConfusionMatrix() {
+ String url = coreURI + CONFUSION_MATRIX_PATH;
+ ConfusionMatrix confusionMatrix = restTemplate.getForObject(url, ConfusionMatrix.class);
+ return confusionMatrix;
+ }
+ private List<EstimatorRegistryParameterDto> exchangeAsMap(String path, HttpMethod httpMethod, Map<String, ?> input,
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> responseType, Map<String, ?> uriVariables) {
+ String url = coreURI + path;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<Map<String, ?>> httpEntity = new HttpEntity<>(input, httpHeaders);
+ return restTemplate.exchange(url, httpMethod, httpEntity, responseType, uriVariables).getBody();
+ }
+}
--- /dev/null
+package hu.user.corecli.service;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+@Service
+public class EstimatorClient {
+
+ private static final String PREDICTION_PATH = "/prediction";
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String ESTIMATOR_DELETE_PATH = "/estimator/{id}";
+ private static final String ID = "id";
+ private static final String ESTIMATOR_ID = "estimator_id";
+
+ @Value("${estimatorUrl}") URI estimatorURI;
+ private RestTemplate restTemplate;
+
+ public EstimatorClient(RestTemplate restTemplate) {
+ this.restTemplate = restTemplate;
+ }
+ public List<Integer> addEstimatorInEstimatorModule(String id) {
+ Map<String, ?> input = Map.of(ESTIMATOR_ID, Integer.parseInt(id));
+ ParameterizedTypeReference<List<Integer>> responseType = new ParameterizedTypeReference<>() {
+ };
+ List<Integer> result = exchangeAsList(ESTIMATOR_PATH, HttpMethod.POST, input, responseType, Collections.emptyMap());
+ return result;
+ }
+ public List<Integer> getEstimatorInEstimatorModule() {
+ ParameterizedTypeReference<List<Integer>> responseType = new ParameterizedTypeReference<>() {
+ };
+ List<Integer> result = exchangeAsList(ESTIMATOR_PATH, HttpMethod.GET, Collections.emptyMap(), responseType, Collections.emptyMap());
+ return result;
+ }
+ public List<Integer> deleteEstimatorInEstimatorModule(String idAsString) {
+ ParameterizedTypeReference<List<Integer>> responseType = new ParameterizedTypeReference<>() {
+ };
+ Map<String, ?> uriParam = Map.of(ID, Integer.parseInt(idAsString));
+ List<Integer> result = exchangeAsList(ESTIMATOR_DELETE_PATH, HttpMethod.DELETE, Collections.emptyMap(), responseType, uriParam);
+ return result;
+ }
+ private <T> List<T> exchangeAsList(String path, HttpMethod httpMethod, Map<String, ?> input,
+ ParameterizedTypeReference<List<T>> responseType, Map<String, ?> uriVariables) {
+ String url = estimatorURI + path;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<Map<String, ?>> httpEntity = new HttpEntity<>(input, httpHeaders);
+ return restTemplate.exchange(url, httpMethod, httpEntity, responseType, uriVariables).getBody();
+ }
+}
--- /dev/null
+server.port=8087
+estimatorUrl=http://mkigui:8083
+coreUrl=http://mkigui:8081
+spring.main.banner-mode=off
+logging.pattern.console=
\ No newline at end of file
--- /dev/null
+server.port=8087
+estimatorUrl=http://localhost:8083
+coreUrl=http://localhost:8081
+
--- /dev/null
+<wsdl:definitions xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/" xmlns:sch="http://spring.io/guides/gs-producing-web-service" xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/" xmlns:tns="http://spring.io/guides/gs-producing-web-service" targetNamespace="http://spring.io/guides/gs-producing-web-service">
+ <wsdl:types>
+ <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://spring.io/guides/gs-producing-web-service">
+ <xs:element name="generateDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="detection_request" type="tns:detection_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="generateGenericDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="unbounded" minOccurs="1" name="detection_request" type="tns:generic_detection_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="storeRealTransactionRequest">
+ <xs:complexType>
+ <xs:sequence maxOccurs="unbounded" minOccurs="1">
+ <xs:element maxOccurs="unbounded" minOccurs="1" name="store_request" type="tns:store_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="storeRealTransactionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="store_result" type="tns:store_result"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="generateDetectionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="detection_response" type="tns:detection_response"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:complexType name="detection_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="card_number" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_type" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="timestamp" type="xs:dateTime"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="amount" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="currency_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="response_code" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="country_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="vendor_code" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="generic_detection_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="field_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="value" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="detection_response">
+ <xs:sequence>
+ <xs:element name="prediction" type="xs:int"/>
+ <xs:element name="negative_probability" type="xs:double"/>
+ <xs:element name="positive_probability" type="xs:double"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="store_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="fraud" type="xs:int"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="store_result">
+ <xs:sequence>
+ <xs:element name="result" type="xs:string"/>
+ <xs:element name="record_number" type="xs:int"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:schema>
+ </wsdl:types>
+ <wsdl:message name="storeRealTransactionResponse">
+ <wsdl:part element="tns:storeRealTransactionResponse" name="storeRealTransactionResponse"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="storeRealTransactionRequest">
+ <wsdl:part element="tns:storeRealTransactionRequest" name="storeRealTransactionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateDetectionRequest">
+ <wsdl:part element="tns:generateDetectionRequest" name="generateDetectionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateDetectionResponse">
+ <wsdl:part element="tns:generateDetectionResponse" name="generateDetectionResponse"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateGenericDetectionRequest">
+ <wsdl:part element="tns:generateGenericDetectionRequest" name="generateGenericDetectionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:portType name="DetectionPort">
+ <wsdl:operation name="storeRealTransaction">
+ <wsdl:input message="tns:storeRealTransactionRequest" name="storeRealTransactionRequest"> </wsdl:input>
+ <wsdl:output message="tns:storeRealTransactionResponse" name="storeRealTransactionResponse"> </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateDetection">
+ <wsdl:input message="tns:generateDetectionRequest" name="generateDetectionRequest"> </wsdl:input>
+ <wsdl:output message="tns:generateDetectionResponse" name="generateDetectionResponse"> </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateGenericDetection">
+ <wsdl:input message="tns:generateGenericDetectionRequest" name="generateGenericDetectionRequest"> </wsdl:input>
+ </wsdl:operation>
+ </wsdl:portType>
+ <wsdl:binding name="DetectionPortSoap11" type="tns:DetectionPort">
+ <soap:binding style="document" transport="http://schemas.xmlsoap.org/soap/http"/>
+ <wsdl:operation name="storeRealTransaction">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="storeRealTransactionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ <wsdl:output name="storeRealTransactionResponse">
+ <soap:body use="literal"/>
+ </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateDetection">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="generateDetectionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ <wsdl:output name="generateDetectionResponse">
+ <soap:body use="literal"/>
+ </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateGenericDetection">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="generateGenericDetectionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ </wsdl:operation>
+ </wsdl:binding>
+ <wsdl:service name="DetectionPortService">
+ <wsdl:port binding="tns:DetectionPortSoap11" name="DetectionPortSoap11">
+ <soap:address location="http://localhost:8081/ws"/>
+ </wsdl:port>
+ </wsdl:service>
+</wsdl:definitions>
\ No newline at end of file
--- /dev/null
+package hu.user.corecli;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class CoreCliApplicationTests {
+
+ @Test
+ void contextLoads() {
+ }
+
+}
--- /dev/null
+FROM adoptopenjdk:14-jre-hotspot
+WORKDIR /opt/app
+COPY target/*.jar coremanager.jar
+CMD ["java","-jar","coremanager.jar"]
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.5.3</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>hu.user</groupId>
+ <artifactId>CoreManager</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>CoreManager</name>
+ <description>CoreManager</description>
+ <properties>
+ <java.version>11</java.version>
+ <zkspringboot.version>2.5.3</zkspringboot.version>
+ </properties>
+
+ <repositories> <!--add-->
+ <repository>
+ <id>spring-snapshots</id>
+ <name>Spring Snapshots</name>
+ <url>https://repo.spring.io/snapshot</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>spring-milestones</id>
+ <name>Spring Milestones</name>
+ <url>https://repo.spring.io/milestone</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>ZK CE</id>
+ <name>ZK CE Repository</name>
+ <url>http://mavensync.zkoss.org/maven2</url>
+ </repository>
+ <repository>
+ <id>ZK EVAL</id>
+ <name>ZK Evaluation Repository</name>
+ <url>http://mavensync.zkoss.org/eval</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web-services</artifactId>
+ </dependency>
+ <dependency> <!--add-->
+ <groupId>org.zkoss.zkspringboot</groupId>
+ <artifactId>zkspringboot-starter</artifactId>
+ <type>pom</type>
+ <version>${zkspringboot.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.zkoss.zk</groupId>
+ <artifactId>zkplus</artifactId>
+ <version>9.6.0-Eval</version>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>31.0.1-jre</version>
+ </dependency>
+ <dependency>
+ <groupId>com.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ <version>4.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+<!-- <dependency>-->
+<!-- <groupId>hu.user</groupId>-->
+<!-- <artifactId>Core</artifactId>-->
+<!-- <version>0.0.1-SNAPSHOT</version>-->
+<!-- <scope>compile</scope>-->
+<!-- </dependency>-->
+ <dependency>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-core</artifactId>
+ <version>4.0.1</version>
+ </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.glassfish.jaxb</groupId>-->
+<!-- <artifactId>jaxb-runtime</artifactId>-->
+<!-- <version>4.0.1</version>-->
+<!-- </dependency>-->
+<!-- <!– https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api –>-->
+<!-- <dependency>-->
+<!-- <groupId>javax.xml.bind</groupId>-->
+<!-- <artifactId>jaxb-api</artifactId>-->
+<!-- <version>2.3.0</version>-->
+<!-- </dependency>-->
+<!-- <!– https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-impl –>-->
+<!-- <dependency>-->
+<!-- <groupId>com.sun.xml.bind</groupId>-->
+<!-- <artifactId>jaxb-impl</artifactId>-->
+<!-- <version>4.0.1</version>-->
+<!-- </dependency>-->
+ <!-- JAXB API only -->
+
+
+<!-- <dependency>-->
+<!-- <groupId>jakarta.xml.bind</groupId>-->
+<!-- <artifactId>jakarta.xml.bind-api</artifactId>-->
+<!-- <version>4.0.0</version>-->
+<!-- </dependency>-->
+
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>4.0.1</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>hu.user</groupId>
+ <artifactId>CommonDto</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>jaxb2-maven-plugin</artifactId>
+ <version>2.5.0</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-value-constructor</artifactId>
+ <version>3.0</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>xjc</id>
+ <goals>
+ <goal>xjc</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceType>wsdl</sourceType>
+ <sources>
+ <source>${project.basedir}/src/main/resources/wsdl/detection.wsdl</source>
+ </sources>
+ <arguments>
+ <argument>-Xvalue-constructor</argument>
+ </arguments>
+ <extension>true</extension>
+ <packageName>hu.user.coremanager</packageName>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
--- /dev/null
+package hu.user.coremanager;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class CoreManagerApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(CoreManagerApplication.class, args);
+ }
+
+}
--- /dev/null
+package hu.user.coremanager.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.oxm.jaxb.Jaxb2Marshaller;
+
+import hu.user.coremanager.service.CoreSoapClient;
+
+@Configuration
+public class CoreSoapClientConfiguration {
+
+ @Bean
+ public Jaxb2Marshaller marshaller() {
+ Jaxb2Marshaller marshaller = new Jaxb2Marshaller();
+ // this package must match the package in the <generatePackage> specified in
+ // pom.xml
+// marshaller.setContextPath("com.example.consumingwebservice.wsdl");
+ marshaller.setContextPath("hu.user.coremanager");
+ return marshaller;
+ }
+
+ @Bean
+ public CoreSoapClient coreSoapClient(Jaxb2Marshaller marshaller) {
+ CoreSoapClient client = new CoreSoapClient();
+ client.setDefaultUri("http://localhost:8081/ws");
+ client.setMarshaller(marshaller);
+ client.setUnmarshaller(marshaller);
+ return client;
+ }
+}
--- /dev/null
+package hu.user.coremanager.config;\r
+\r
+import javax.annotation.PostConstruct;\r
+\r
+import org.slf4j.Logger;\r
+import org.slf4j.LoggerFactory;\r
+import org.springframework.context.annotation.Configuration;\r
+import org.springframework.context.annotation.Profile;\r
+import org.zkoss.lang.Library;\r
+import org.zkoss.zk.ui.WebApps;\r
+\r
+@Configuration\r
+@Profile("dev")\r
+public class DevelopmentConfig {\r
+ private static Logger logger = LoggerFactory.getLogger(DevelopmentConfig.class);\r
+\r
+ @PostConstruct\r
+ public void initDevelopmentProperties() throws Exception {\r
+ logger.info("**************************************************************");\r
+ logger.info("**** ZK-Springboot-Demo: development configuration active ****");\r
+ logger.info("**************************************************************");\r
+\r
+ //disable various caches to avoid server restarts\r
+ Library.setProperty("org.zkoss.zk.ZUML.cache", "false");\r
+ Library.setProperty("org.zkoss.zk.WPD.cache", "false");\r
+ Library.setProperty("org.zkoss.zk.WCS.cache", "false");\r
+ Library.setProperty("org.zkoss.web.classWebResource.cache", "false");\r
+ Library.setProperty("org.zkoss.util.label.cache", "false");\r
+\r
+ // enable non minified js\r
+ WebApps.getCurrent().getConfiguration().setDebugJS(true);\r
+\r
+ // enable for debugging MVVM commands and binding (very verbose)\r
+ Library.setProperty("org.zkoss.bind.DebuggerFactory.enable", "false");\r
+ }\r
+}\r
--- /dev/null
+package hu.user.coremanager.config;
+
+import java.time.Duration;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+public class RestConfig {
+
+ @Bean
+ public RestTemplate getRestTemplate(RestTemplateBuilder builder) {
+ RestTemplate restTemplate = builder
+ .setReadTimeout(Duration.ofSeconds(7))
+ .build();
+ return restTemplate;
+
+ }
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class ConfusionMatrixDto {
+
+ private int tp;
+ private int tn;
+ private int fp;
+ private int fn;
+
+
+
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import java.io.File;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class DirectoryProperty {
+
+ private String displayName;
+ private File directory;
+
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+
+@Getter
+public class DownStreamErrorDto {
+
+ private int errorCode;
+ private String errorName;
+ private String errorMessage;
+
+ public DownStreamErrorDto(@JsonProperty(value = "error_code", required = true) int errorCode,
+ @JsonProperty(value = "error_name", required = true) String errorName,
+ @JsonProperty(value = "error_message", required = true) String errorMessage) {
+ this.errorCode = errorCode;
+ this.errorName = errorName;
+ this.errorMessage = errorMessage;
+ }
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class EstimatorMetricsDto {
+ private int id;
+ private int tp;
+ private int tn;
+ private int fp;
+ private int fn;
+ private double sensitivity;
+ private double specifity;
+ private double accuracy;
+ private double balancedAccuracy;
+ private double prec;
+ private double recall;
+ private double ppv;
+ private double npv;
+ private double fnr;
+ private double fpr;
+ private double fdr;
+ private double _for;
+ private double f1;
+ private double f05;
+ private double f2;
+ private double mcc;
+ private double rocauc;
+ private double youden;
+
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class EstimatorParameter {
+
+ private int estimatorId;
+ private int weight;
+
+ public EstimatorParameter(@JsonProperty(value = "estimator_id",required = true) int estimatorId, int weight) {
+ this.estimatorId = estimatorId;
+ this.weight = weight;
+ }
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class FieldProperty {
+
+ private String fieldName;
+ private String fieldType;
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import java.io.File;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class FileProperty {
+
+ private File file;
+ private String fileName;
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class GenericRecordCollectionDto {
+ private List<String> fieldTypes = new ArrayList<>();
+ private List<String> fieldNames = new ArrayList<>();
+ private List<List<String>> records = new ArrayList<>();
+
+ // @formatter:off
+ public GenericRecordCollectionDto(
+ @JsonProperty(value = "field_types", required = true) List<String> fieldTypes,
+ @JsonProperty(value = "field_names", required = true) List<String> fieldNames,
+ @JsonProperty(value = "records", required = true) List<List<String>> records) {
+ this.fieldTypes = fieldTypes;
+ this.fieldNames = fieldNames;
+ this.records = records;
+ }
+ // @formatter:on
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class MetricsAndTrainTaskName {
+
+ private String trainTaskName;
+ private MetricsDto metricsDto;
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class MetricsDto {
+
+ private int id;
+
+ private int estimatorId;
+ private int truePositive;
+ private int trueNegative;
+ private int falsePositive;
+ private int falseNegative;
+ private double accuracy;
+ private double balancedAccuracy;
+ private double sensitivity;
+ private double specifity;
+ private double prec;
+ private double recall;
+ private double ppv;
+ private double npv;
+ private double fnr;
+ private double fpr;
+ private double fdr;
+ private double _for;
+ private double f1;
+ private double f05;
+ private double f2;
+ private double mcc;
+ private double rocauc;
+ private double youden;
+
+ // @formatter:off
+ public MetricsDto(
+ @JsonProperty(value = "id", required = true) int id,
+ @JsonProperty(value = "estimator_id", required = true) int estimatorId,
+ @JsonProperty(value = "TP", required = true) int truePositive,
+ @JsonProperty(value = "TN", required = true) int trueNegative,
+ @JsonProperty(value = "FP", required = true) int falsePositive,
+ @JsonProperty(value = "FN", required = true) int falseNegative,
+ @JsonProperty(value = "accuracy", required = true) double accuracy,
+ @JsonProperty(value = "balanced_accuracy", required = true) double balancedAccuracy,
+ @JsonProperty(value = "sensitivity", required = true) double sensitivity,
+ @JsonProperty(value = "specificity", required = true) double specifity,
+ @JsonProperty(value = "precision", required = true) double prec,
+ @JsonProperty(value = "recall", required = true) double recall,
+ @JsonProperty(value = "PPV", required = true) double ppv,
+ @JsonProperty(value = "NPV", required = true) double npv,
+ @JsonProperty(value = "FNR", required = true) double fnr,
+ @JsonProperty(value = "FPR", required = true) double fpr,
+ @JsonProperty(value = "FDR", required = true) double fdr,
+ @JsonProperty(value = "FOR", required = true) double _for,
+ @JsonProperty(value = "f1", required = true) double f1,
+ @JsonProperty(value = "f0.5", required = true) double f05,
+ @JsonProperty(value = "f2", required = true) double f2,
+ @JsonProperty(value = "MCC", required = true) double mcc,
+ @JsonProperty(value = "ROCAUC", required = true) double rocauc,
+ @JsonProperty(value = "Youdens_statistic", required = true) double youden)
+ // @formatter:on
+ {
+ this.id = id;
+ this.estimatorId = estimatorId;
+ this.truePositive = truePositive;
+ this.trueNegative = trueNegative;
+ this.falsePositive = falsePositive;
+ this.falseNegative = falseNegative;
+ this.accuracy = accuracy;
+ this.balancedAccuracy = balancedAccuracy;
+ this.sensitivity = sensitivity;
+ this.specifity = specifity;
+ this.prec = prec;
+ this.recall = recall;
+ this.ppv = ppv;
+ this.npv = npv;
+ this.fnr = fnr;
+ this.fpr = fpr;
+ this.fdr = fdr;
+ this._for = _for;
+ this.f1 = f1;
+ this.f05 = f05;
+ this.f2 = f2;
+ this.mcc = mcc;
+ this.rocauc = rocauc;
+ this.youden = youden;
+ }
+}
--- /dev/null
+package hu.user.coremanager.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class ObservedValue {
+
+ private String uuid;
+ private int observedFraud;
+}
--- /dev/null
+package hu.user.coremanager.exception;
+
+public class ParametersExistYetException extends RuntimeException{
+
+ private String errorMessage;
+
+ public ParametersExistYetException(String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.common.dto.ConfusionMatrixDto;
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.coremanager.dto.ObservedValue;
+
+@Service
+public class CoreClient {
+
+ @Value("${coreUrl}") String coreUrl;
+ @Autowired private RestTemplate restTemplate;
+
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String OBSERVED_PATH = "/observed";
+ private static final String TRANSACTIONM_PATH = "/transaction";
+ private static final String CONFUSION_MATRIX_PATH = "/confusion_matrix";
+ private static final String TRANSACTION_CLEAR_PATH = "/transaction_clear";
+
+ private static final String ID = "id";
+
+ public List<EstimatorRegistryParameterDto> registryEstimator(EstimatorRegistryParameterDto estimatorRegistryParameterDto) {
+ String url = coreUrl + ESTIMATOR_PATH;
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<EstimatorRegistryParameterDto> httpEntity = new HttpEntity<>(estimatorRegistryParameterDto, httpHeaders);
+ ResponseEntity<List<EstimatorRegistryParameterDto>> responseFromCoreModul =
+ restTemplate.exchange(url, HttpMethod.POST, httpEntity, typeReference, Collections.emptyMap());
+ List<EstimatorRegistryParameterDto> registeredEstimatorDtos = responseFromCoreModul.getBody();
+ return registeredEstimatorDtos;
+ }
+
+ public List<EstimatorRegistryParameterDto> getRegisteredEstimators() {
+ String url = coreUrl + ESTIMATOR_PATH;
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<EstimatorRegistryParameterDto> httpEntity = new HttpEntity<>(httpHeaders);
+ ResponseEntity<List<EstimatorRegistryParameterDto>> responseFromCoreModul =
+ restTemplate.exchange(url, HttpMethod.GET, httpEntity, typeReference);
+ List<EstimatorRegistryParameterDto> registeredEstimatorDtos = responseFromCoreModul.getBody();
+ return registeredEstimatorDtos;
+ }
+
+ public List<EstimatorRegistryParameterDto> deleteEstimator(Integer id) {
+ String url = coreUrl + ESTIMATOR_PATH + "/{id}";
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> typeReference = new ParameterizedTypeReference<>() {
+ };
+
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<EstimatorRegistryParameterDto> httpEntity = new HttpEntity<>(httpHeaders);
+ ResponseEntity<List<EstimatorRegistryParameterDto>> responseFromCoreModul =
+ restTemplate.exchange(url, HttpMethod.DELETE, httpEntity, typeReference, Map.of(ID, id));
+ List<EstimatorRegistryParameterDto> registeredEstimatorDtos = responseFromCoreModul.getBody();
+ return registeredEstimatorDtos;
+ }
+
+ public List<EstimatorRegistryParameterDto> modifyEstimatorWeight(EstimatorRegistryParameterDto estimatorRegistryParameterDto) {
+ String url = coreUrl + ESTIMATOR_PATH;
+ ParameterizedTypeReference<List<EstimatorRegistryParameterDto>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<EstimatorRegistryParameterDto> httpEntity = new HttpEntity<>(estimatorRegistryParameterDto, httpHeaders);
+ ResponseEntity<List<EstimatorRegistryParameterDto>> responseFromCoreModul =
+ restTemplate.exchange(url, HttpMethod.PUT, httpEntity, typeReference, Collections.emptyMap());
+ List<EstimatorRegistryParameterDto> registeredEstimatorDtos = responseFromCoreModul.getBody();
+ return registeredEstimatorDtos;
+ }
+
+ public ConfusionMatrixDto getSysTemConfusionMatrix() {
+ String url = coreUrl + CONFUSION_MATRIX_PATH;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ ResponseEntity<ConfusionMatrixDto> responseFromCoreModul = restTemplate.getForEntity(url, ConfusionMatrixDto.class);
+ return responseFromCoreModul.getBody();
+ }
+
+ public boolean uploadToCoreModuleObservedValues(List<ObservedValue> observedValues) {
+ String url = coreUrl + OBSERVED_PATH;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity<List<ObservedValue>> httpEntity = new HttpEntity<>(observedValues,httpHeaders);
+ ResponseEntity<?> responseFromCoreModul =
+ restTemplate.exchange(url, HttpMethod.POST, httpEntity, ResponseEntity.class, Collections.emptyMap());
+ if (responseFromCoreModul.getStatusCode().is2xxSuccessful()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ public int getAllObservedTransaction() {
+ String url = coreUrl + OBSERVED_PATH;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ ResponseEntity<Integer> responseFromCoreModul = restTemplate.getForEntity(url, Integer.class);
+ return responseFromCoreModul.getBody();
+ }
+ public int getAllTransaction() {
+ String url = coreUrl + TRANSACTIONM_PATH;
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ ResponseEntity<Integer> responseFromCoreModul = restTemplate.getForEntity(url, Integer.class);
+ return responseFromCoreModul.getBody();
+ }
+
+ public void clearAllTransaction() {
+ String url = coreUrl + TRANSACTION_CLEAR_PATH;
+ ResponseEntity<?> responseFromCoreModul = restTemplate.postForEntity(url, null, null, Collections.emptyMap());
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.ws.client.core.support.WebServiceGatewaySupport;
+import org.springframework.ws.soap.client.core.SoapActionCallback;
+
+import hu.user.coremanager.DetectionResponse;
+import hu.user.coremanager.GenerateDetectionResponse;
+import hu.user.coremanager.GenerateGenericDetectionRequest;
+
+public class CoreSoapClient extends WebServiceGatewaySupport {
+
+ @Value("${coreSoapUri}") String coreSoapUri;
+
+ public DetectionResponse getResponse(GenerateGenericDetectionRequest request) {
+
+
+ GenerateDetectionResponse response = (GenerateDetectionResponse) getWebServiceTemplate()
+ .marshalSendAndReceive(coreSoapUri, request,
+ new SoapActionCallback(
+ "http://spring.io/guides/gs-producing-web-service/generateGenericDetectionRequest"));
+
+ return response.getDetectionResponse();
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.util.Arrays;
+import java.util.EmptyStackException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.coremanager.dto.EstimatorParameter;
+
+@Service
+public class EstimatorClient {
+
+ private static final String ID = "id";
+
+ @Value("${estimatorUrl}") String estimatorUrl;
+ @Autowired private RestTemplate restTemplate;
+
+ private static final String ESTIMATOR_PATH = "/estimator";
+
+ public List<Integer> registerEstimator(EstimatorRegistryParameterDto estimatorRegistryParameterDto) {
+ String url = estimatorUrl + ESTIMATOR_PATH;
+ int[] result = restTemplate.postForEntity(url, estimatorRegistryParameterDto, int[].class).getBody();
+ return Arrays.stream(result).boxed().collect(Collectors.toList());
+ }
+
+ public List<Integer> deleteEstimator(int id) {
+ String url = estimatorUrl + ESTIMATOR_PATH + "/{id}";
+ int[] result = restTemplate.exchange(url, HttpMethod.DELETE, null, int[].class, Map.of(ID, id)).getBody();
+ return Arrays.stream(result).boxed().collect(Collectors.toList());
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import hu.user.coremanager.DetectionResponse;
+import hu.user.coremanager.GenerateGenericDetectionRequest;
+import hu.user.coremanager.GenericDetectionRequest;
+import hu.user.coremanager.dto.GenericRecordCollectionDto;
+
+@Service
+public class EvaluationService {
+ private static final String SCHEMA_NAME = "schema_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String PREDICTED_POSITIVE = "predicted_positive";
+ private static final String PREDICTED_NEGATIVE = "predicted_negative";
+
+ private TrainClient trainClient;
+ private CoreSoapClient coreSoapClient;
+
+ public EvaluationService(TrainClient trainClient, CoreSoapClient coreSoapClient) {
+
+ this.trainClient = trainClient;
+ this.coreSoapClient = coreSoapClient;
+ }
+
+ public Map<String,Integer> predictForEvaluation(String schema, String table) {
+ int positivePredictionCaseNumber = 0;
+ int negativePredictionCaseNumber = 0;
+ GenericRecordCollectionDto genericRecordCollectionDto =
+ trainClient.getGenericRecordsForEvaluation(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ List<List<String>> records = genericRecordCollectionDto.getRecords();
+ List<String> fieldNames = genericRecordCollectionDto.getFieldNames();
+ int fieldNumber = fieldNames.size();
+ List<GenericDetectionRequest> genericDetectionRequests = new ArrayList<>();
+ for (List<String> record : records) {
+ for (int i = 0; i < fieldNumber; i++) {
+ GenericDetectionRequest genericDetectionRequest =
+ new GenericDetectionRequest(fieldNames.get(i), record.get(i));
+ genericDetectionRequest.setFieldName(fieldNames.get(i));
+ genericDetectionRequests.add(genericDetectionRequest);
+ }
+ GenerateGenericDetectionRequest generateGenericDetectionRequest =
+ new GenerateGenericDetectionRequest(genericDetectionRequests);
+
+ DetectionResponse response = coreSoapClient.getResponse(generateGenericDetectionRequest);
+ if (response.getPrediction() == 0) {
+ negativePredictionCaseNumber++;
+ } else {
+ positivePredictionCaseNumber++;
+ }
+ System.out.println("Prediction: " + response.getPrediction());
+ System.out.println("Positive probability: " + response.getPositiveProbability());
+ System.out.println("Negatíve probability: " + response.getNegativeProbability());
+
+ }
+ Map<String, Integer> result = Map.of(PREDICTED_POSITIVE, positivePredictionCaseNumber, PREDICTED_NEGATIVE,
+ negativePredictionCaseNumber);
+ return result;
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+
+import hu.user.coremanager.dto.ObservedValue;
+
+@Service
+public class FileHandler {
+
+ public List<ObservedValue> readObservedValueFromCsvFile(File csvFile) {
+ List<ObservedValue> observedValues = new ArrayList<>();
+ try {
+ FileReader filereader = new FileReader(csvFile);
+ CSVReader csvReader = new CSVReaderBuilder(filereader).withSkipLines(1).build();
+ List<String[]> allData = csvReader.readAll();
+ for (String[] row : allData) {
+ ObservedValue observedValue = new ObservedValue(row[0], Integer.valueOf(row[1]));
+ observedValues.add(observedValue);
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return observedValues;
+ }
+}
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.annotation.ScopedProxyMode;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.coremanager.dto.ConfusionMatrixDto;
+import hu.user.coremanager.dto.GenericRecordCollectionDto;
+import hu.user.coremanager.dto.MetricsDto;
+import hu.user.coremanager.exception.ParametersExistYetException;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS)
+@Slf4j
+public class TrainClient {
+
+ private static final String OK = "OK";
+ private static final String ENCODING_AND_FEATURE_ENGINEERING_PATH = "/encoding_and_feature_engineering_plan";
+ private static final String TRAIN_TASK_PATH = "/train_task";
+ private static final String FIT_PATH = "/fit";
+ private static final String SCHEMA_PATH = "/schema";
+ private static final String TABLE_PATH = "/table";
+ private static final String FIELD_NAME_AND_TYPE_PATH = "/field_name_and_type";
+ private static final String FIELD_NAME_IN_ORIDNAL_POSITION_PATH = "/field_name_in_ordinal_position";
+ private static final String FIELD_TYPE_IN_ORIDNAL_POSITION_PATH = "/field_type_in_ordinal_position";
+ private static final String RECORD_PATH = "/record";
+ private static final String METRICS_PATH = "/metrics";
+ private static final String CPU_PATH = "/cpu";
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String CONFUSION_MATRIX_PATH = "/confusion_matrix";
+ private static final String LIVENESS_PATH = "/liveness";
+ private static final String TABLE_INFO_PATH = "/table_info";
+ private static final String GENERIC_RECORD_PATH = "/generic_records";
+ private static final String TRAIN_TASK_NAME_BY_ESTIMATOR_ID_PATH = "/train_task_name_by_estimator_id";
+
+ @Value("${trainUrl}") String trainUrl;
+ @Value("${home.directory}") private String homeDirectory;
+ @Autowired private RestTemplate restTemplate;
+
+// public TrainModuleStatus testLiveness() {
+// TrainModuleStatus trainModuleStatus = null;
+// String url = trainUrl + LIVENESS_PATH;
+// try {
+// LivenessDto livenessDto = restTemplate.getForEntity(url, LivenessDto.class).getBody();
+// if (livenessDto.getResponse().equals(OK)) {
+// trainModuleStatus = TrainModuleStatus.OK;
+// }
+// } catch (ResourceAccessException exception) {
+// if (exception.getCause() instanceof ConnectException) {
+// trainModuleStatus = TrainModuleStatus.NOT_ACCESSIBLE;
+// }
+// if (exception.getCause() instanceof SocketTimeoutException) {
+// //TO-DO logging
+// trainModuleStatus = TrainModuleStatus.RESPONSE_TIMEOUT;
+// }
+// }
+// return trainModuleStatus;
+// }
+
+ public String getHomeDirectory() {
+ return homeDirectory;
+ }
+
+ public int getCpuCoreNumber() {
+ String url = trainUrl + CPU_PATH;
+ int coreNumber = restTemplate.getForEntity(url, Integer.class).getBody();
+ return coreNumber;
+ }
+
+ public String[] getSchemas() {
+ String url = trainUrl + SCHEMA_PATH;
+ String[] schemas = restTemplate.getForEntity(url, String[].class).getBody();
+ return schemas;
+ }
+
+ public String[] getTables(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TABLE_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public String[] getFieldNameInOrdinalPosition(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_NAME_IN_ORIDNAL_POSITION_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+ public String[] getFieldTypeInOrdinalPosition(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_TYPE_IN_ORIDNAL_POSITION_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public Map<String, String> getFieldTypeByName(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_NAME_AND_TYPE_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<Map<String, String>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ Map<String, String> result =
+ restTemplate.exchange(urlTemplate, HttpMethod.GET, null, typeReference, params).getBody();
+ return result;
+ }
+
+ public String getRecordsAsString(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + RECORD_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+ public List<List<Object>> getRecords(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + RECORD_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<List<List<Object>>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ List<List<Object>> records = exchangeAsList(urlTemplate, typeReference, params);
+ return records;
+ }
+
+ public Map<String, String> getFeatureSelectors() {
+ return useCommmonQuery("/available_feature_selector");
+ }
+
+ public Map<String, String> getSamplers() {
+ return useCommmonQuery("/available_sampler");
+ }
+
+ public Map<String, String> getScalers() {
+ return useCommmonQuery("/available_scaler");
+ }
+
+ public Map<String, String> getModels() {
+ return useCommmonQuery("/available_model");
+ }
+
+ public Map<String, String> useCommmonQuery(String path) {
+ String url = trainUrl + path;
+ Map<String, String> parameters = restTemplate.getForEntity(url, Map.class).getBody();
+ return parameters;
+ }
+
+ public List<MetricsDto> getMetrics(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + METRICS_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<List<MetricsDto>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ List<MetricsDto> metrics = exchangeAsList(urlTemplate, typeReference, params);
+ return metrics;
+ }
+
+ public ConfusionMatrixDto getConfusionMatrixElements(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + CONFUSION_MATRIX_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<ConfusionMatrixDto> responseEntity =
+ restTemplate.getForEntity(urlTemplate, ConfusionMatrixDto.class, params);
+ return responseEntity.getBody();
+ }
+
+ public GenericRecordCollectionDto getGenericRecordsForEvaluation(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + GENERIC_RECORD_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<GenericRecordCollectionDto> responseEntity =
+ restTemplate.getForEntity(urlTemplate, GenericRecordCollectionDto.class, params);
+ return responseEntity.getBody();
+ }
+
+ public String getTrainTaskNameByEstimatorId(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TRAIN_TASK_NAME_BY_ESTIMATOR_ID_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> responseEntity =
+ restTemplate.getForEntity(urlTemplate, String.class, params);
+ return responseEntity.getBody();
+ }
+
+
+ public <T> Integer addGenericParameters(T parameters, String path) {
+ String url = trainUrl + path;
+ try {
+ ResponseEntity<Integer> responseFromTrainModule =
+ restTemplate.postForEntity(url, parameters, Integer.class);
+ return responseFromTrainModule.getBody();
+ } catch (HttpClientErrorException httpClientErrorException) {
+ String errorMessage = httpClientErrorException.getResponseBodyAsString();
+ log.error(errorMessage);
+ throw new RuntimeException(errorMessage);
+ }
+ }
+
+ private <T> List<T> exchangeAsList(String uri, ParameterizedTypeReference<List<T>> responseType,
+ Map<String, String> uriVariables) {
+ return restTemplate.exchange(uri, HttpMethod.GET, null, responseType, uriVariables).getBody();
+ }
+
+}
+
--- /dev/null
+package hu.user.coremanager.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.web.util.UriComponentsBuilder;
+
+public class UrlBuilder {
+
+ private String url;
+ private Map<String, String> queryParams = new HashMap<>();
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Map<String, String> getQueryParams() {
+ return queryParams;
+ }
+
+ public void build(String url, Map<String, String> queryValueByParamName) {
+ List<String> paramNames = new ArrayList<>();
+ List<String> paramValues = new ArrayList<>();
+ for (var entry : queryValueByParamName.entrySet()) {
+ paramNames.add(entry.getKey());
+ paramValues.add(entry.getValue());
+ }
+ buildUrl(url, paramNames);
+ fillQueryParams(paramNames, paramValues);
+ }
+
+ private void buildUrl(String url, List<String> queryParamNames) {
+ String urlTemplate;
+ UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromHttpUrl(url);
+ for (String queryParamName : queryParamNames) {
+ uriComponentsBuilder = uriComponentsBuilder.queryParam(queryParamName, "{" + queryParamName + "}");
+ }
+ this.url = uriComponentsBuilder.encode().toUriString();
+ }
+
+ private void fillQueryParams(List<String> paramNames, List<String> paramValuee) {
+ Map<String, String> params = new HashMap<>();
+ for (int i = 0; i < paramNames.size(); i++) {
+ params.put(paramNames.get(i), paramValuee.get(i));
+ }
+ this.queryParams = params;
+ }
+}
--- /dev/null
+package hu.user.coremanager.viewmodel;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.zkoss.bind.BindUtils;
+import org.zkoss.bind.annotation.BindingParam;
+import org.zkoss.bind.annotation.Command;
+import org.zkoss.bind.annotation.Init;
+import org.zkoss.bind.annotation.NotifyChange;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.ListModelList;
+
+import hu.user.common.dto.EstimatorRegistryParameterDto;
+import hu.user.coremanager.dto.MetricsAndTrainTaskName;
+import hu.user.coremanager.dto.MetricsDto;
+import hu.user.coremanager.service.CoreClient;
+import hu.user.coremanager.service.EstimatorClient;
+import hu.user.coremanager.service.TrainClient;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class EstimatorRegistryViewModel {
+
+ private static final String ESTIMATOR_ID = "estimator_id";
+
+ @WireVariable private TrainClient trainClient;
+ @WireVariable private CoreClient coreClient;
+ @WireVariable private EstimatorClient estimatorClient;
+
+ private Integer chosenEstimatorId;
+ private List<Integer> availableWeights = new ArrayList<>();
+ private int currentWeight;
+
+ private int truePositive;
+ private int trueNegative;
+ private int falsePositive;
+ private int falseNegative;
+
+ private Map<Integer, Integer> registeredEstimatorWeightById = new HashMap<>();
+ private ListModelList<EstimatorRegistryParameterDto> estimatorRegistryParameters = new ListModelList<>();
+
+ private ListModelList<MetricsDto> metrics = new ListModelList<>();
+ private ListModelList<MetricsAndTrainTaskName> metricsAndTrainTaskNames;
+
+ private Integer selected;
+
+ private Integer chosenWeight;
+
+ private boolean registryButtonDisabled;
+ private boolean updateButtonDisabled;
+ private boolean enablePredictionButtonDisabled;
+ private boolean disablePredictionButtonDisabled;
+
+ @Init
+ public void init() {
+ registryButtonDisabled = true;
+ fillMetrics();
+ availableWeights = IntStream.rangeClosed(1, 8).boxed().collect(Collectors.toList());
+ List<EstimatorRegistryParameterDto> responseFromCore =
+ coreClient.getRegisteredEstimators();
+ estimatorRegistryParameters = new ListModelList<>(responseFromCore);
+ if (!responseFromCore.isEmpty()) {
+ BindUtils.postGlobalCommand(null, null, "enableTestPredictionStartButton", null);
+ }
+ }
+
+ public void fillMetrics() {
+ List<MetricsAndTrainTaskName> metricsAndTrainTaskNameCollection = new ArrayList<>();
+ List<MetricsDto> metricsDtos = trainClient.getMetrics(Collections.emptyMap());
+ for (MetricsDto metricsDto : metricsDtos) {
+ int estimatorId = metricsDto.getEstimatorId();
+ String trainTaskName =
+ trainClient.getTrainTaskNameByEstimatorId(Map.of(ESTIMATOR_ID, String.valueOf(estimatorId)));
+ MetricsAndTrainTaskName metricsAndTrainTaskName = new MetricsAndTrainTaskName(trainTaskName, metricsDto);
+ metricsAndTrainTaskNameCollection.add(metricsAndTrainTaskName);
+ }
+ metricsAndTrainTaskNames = new ListModelList<>(metricsAndTrainTaskNameCollection);
+ }
+
+ @Command
+ @NotifyChange({"chosenEstimatorId", "truePositive", "trueNegative", "falsePositive", "falseNegative"})
+ public void selectEstimatorId(@BindingParam("param") int selectedEstimatorId) {
+ for (MetricsAndTrainTaskName metricsAndTrantaskName : metricsAndTrainTaskNames) {
+ if (metricsAndTrantaskName.getMetricsDto().getId() == selectedEstimatorId) {
+ truePositive = metricsAndTrantaskName.getMetricsDto().getTruePositive();
+ trueNegative = metricsAndTrantaskName.getMetricsDto().getTrueNegative();
+ falsePositive = metricsAndTrantaskName.getMetricsDto().getFalsePositive();
+ falseNegative = metricsAndTrantaskName.getMetricsDto().getFalseNegative();
+ }
+ if (!estimatorRegistryParameters.stream().map(e -> e.getEstimatorId()).collect(Collectors.toList())
+ .contains(selectedEstimatorId)) {
+ chosenEstimatorId = selectedEstimatorId;
+ }
+ }
+// estimatorId = metrics.get(selected).getId();
+ }
+
+ @Command
+ @NotifyChange({"estimatorRegistryParameters", "chosenEstimatorId", "chosenWeight"})
+ public void registryEstimator() {
+ EstimatorRegistryParameterDto estimatorRegistryParameterDto =
+ new EstimatorRegistryParameterDto(chosenEstimatorId, chosenWeight);
+ List<EstimatorRegistryParameterDto> responseFromCore =
+ coreClient.registryEstimator(estimatorRegistryParameterDto);
+ estimatorRegistryParameters = new ListModelList<>(responseFromCore);
+ responseFromCore.stream().map(e -> "id: " + e.getEstimatorId() + " weight: " + e.getWeight())
+ .collect(Collectors.toList()).forEach(System.out::println);
+ List<Integer> responseFromEstimator = estimatorClient.registerEstimator(estimatorRegistryParameterDto);
+ responseFromEstimator.forEach(System.out::println);
+ if (!responseFromEstimator.isEmpty()) {
+ BindUtils.postGlobalCommand(null, null, "enableTestPredictionStartButton", null);
+ }
+ chosenEstimatorId = null;
+ chosenWeight = null;
+ }
+
+ @Command
+ public void increaseWeight(@BindingParam("param") int chosenEstimatorId) {
+ estimatorRegistryParameters.stream().filter(e -> e.getEstimatorId() == chosenEstimatorId).findFirst()
+ .ifPresent(this::increaseEstimatorWeight);
+ }
+
+ @Command
+ public void decreaseWeight(@BindingParam("param") int chosenEstimatorId) {
+ estimatorRegistryParameters.stream().filter(e -> e.getEstimatorId() == chosenEstimatorId).findFirst()
+ .ifPresent(this::decreaseEstimatorWeight);
+ }
+
+ private void increaseEstimatorWeight(EstimatorRegistryParameterDto parameterDto) {
+ int weigth = parameterDto.getWeight();
+ parameterDto.setWeight(weigth + 1);
+ estimatorRegistryParameters = new ListModelList<>(coreClient.modifyEstimatorWeight(parameterDto));
+ BindUtils.postNotifyChange(null, null, this, "estimatorRegistryParameters");
+ }
+
+ private void decreaseEstimatorWeight(EstimatorRegistryParameterDto parameterDto) {
+ int weigth = parameterDto.getWeight();
+ if (weigth > 1) {
+ parameterDto.setWeight(weigth - 1);
+ estimatorRegistryParameters = new ListModelList<>(coreClient.modifyEstimatorWeight(parameterDto));
+ BindUtils.postNotifyChange(null, null, this, "estimatorRegistryParameters");
+ }
+ }
+
+ @Command
+ @NotifyChange("estimatorRegistryParameters")
+ public void deleteEstimator(@BindingParam("param") int chosenEstimatorId) {
+ estimatorRegistryParameters = new ListModelList<>(coreClient.deleteEstimator(chosenEstimatorId));
+ List<Integer> registeredEstimatorIdEstimatorModule = estimatorClient.deleteEstimator(chosenEstimatorId);
+ if (registeredEstimatorIdEstimatorModule.isEmpty()) {
+ BindUtils.postGlobalCommand(null, null, "disableTestPredictionStartButton", null);
+ }
+ }
+}
--- /dev/null
+package hu.user.coremanager.viewmodel;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.yaml.snakeyaml.events.Event;
+import org.zkoss.bind.annotation.BindingParam;
+import org.zkoss.bind.annotation.GlobalCommand;
+import org.zkoss.bind.annotation.Init;
+import org.zkoss.bind.annotation.NotifyChange;
+import org.zkoss.zk.ui.annotation.Command;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.Messagebox;
+
+import com.google.common.io.Files;
+
+import hu.user.common.dto.ConfusionMatrixDto;
+import hu.user.coremanager.dto.DirectoryProperty;
+import hu.user.coremanager.dto.FieldProperty;
+import hu.user.coremanager.dto.FileProperty;
+import hu.user.coremanager.dto.ObservedValue;
+import hu.user.coremanager.service.CoreClient;
+import hu.user.coremanager.service.EvaluationService;
+import hu.user.coremanager.service.FileHandler;
+import hu.user.coremanager.service.TrainClient;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class QualifyingViewModel {
+
+ private static final String ID = "id";
+ private static final String TRANSACTION_IDENTIFIER = "transaction_identifier";
+ private static final String CSV = "csv";
+ private static final String SCHEMA_NAME = "schema_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String PREDICTED_POSITIVE = "predicted_positive";
+ private static final String PREDICTED_NEGATIVE = "predicted_negative";
+ private static final String REAL = "real";
+ private static final String EVALUATION = "evaluation";
+
+ @WireVariable private FileHandler fileHandler;
+ @WireVariable private CoreClient coreClient;
+ @WireVariable private TrainClient trainClient;
+ @WireVariable private EvaluationService evaluationService;
+ private String testTransactionSchema;
+ private String realTransactionSchema;
+ private List<String> schemas = new ArrayList<>();
+ private String testTransactiontable;
+ private String realTransactiontable;
+ private List<String> testTransactionTables = new ArrayList<>();
+ private List<String> realTransactionTables = new ArrayList<>();
+ private List<FieldProperty> fieldProperties = new ArrayList<>();
+ private List<File> folders = new ArrayList<>();
+ private List selectedFolders;
+ private List<File> files = new ArrayList<>();
+ private List<String> fileNames = new ArrayList<>();
+ private File selectedFolder;
+ private String fullPath;
+ private List<FileProperty> fileProperties = new ArrayList<>();
+ private List<DirectoryProperty> directoryProperties = new ArrayList<>();
+
+ private Integer selectedFileNameIndex;
+ private ConfusionMatrixDto confusionMatrixDto;
+
+ private Integer positivePredictionResultForEvaluation;
+ private Integer negativePredictionResultForEvaluation;
+ private String homeDirectory;
+ private boolean disabledTestPredictionStartButton;
+
+ @Init
+ public void init() {
+ disabledTestPredictionStartButton = true;
+ schemas = Arrays.stream(trainClient.getSchemas()).collect(Collectors.toList());
+ homeDirectory = trainClient.getHomeDirectory();
+ File root = new File(homeDirectory);
+ directoryProperties = getDirectoryProperties(root);
+ fullPath = root.getAbsolutePath();
+ }
+
+ public String getExtensionByGuava(String filename) {
+ return Files.getFileExtension(filename);
+ }
+
+ @Command
+ @NotifyChange("testTransactionTables")
+ public void changeTestTransactionSchema() {
+ if (testTransactionSchema != null && schemas.contains(testTransactionSchema)) {
+ String[] tablesOfSelectedSchema = trainClient.getTables(Map.of(SCHEMA_NAME, testTransactionSchema));
+ testTransactionTables = Arrays.stream(tablesOfSelectedSchema).filter(e -> e.contains(EVALUATION))
+ .collect(Collectors.toList());
+ }
+ }
+
+ @Command
+ @NotifyChange("realTransactionTables")
+ public void changeRealTransactionSchema() {
+ if (realTransactionSchema != null && schemas.contains(realTransactionSchema)) {
+ String[] tablesOfSelectedSchema = trainClient.getTables(Map.of(SCHEMA_NAME, realTransactionSchema));
+ realTransactionTables =
+ Arrays.stream(tablesOfSelectedSchema).filter(e -> e.contains(REAL)).collect(Collectors.toList());
+ }
+ }
+
+ @Command
+ @NotifyChange("fieldProperties")
+ public void changeRealTransactionTable() {
+ Map<String, String> fieldTypeByName = trainClient.getFieldTypeByName(
+ Map.of(SCHEMA_NAME, realTransactionSchema, TABLE_NAME, realTransactiontable));
+ fieldProperties = fieldTypeByName.entrySet().stream().map(e -> new FieldProperty(e.getKey(), e.getValue()))
+ .collect(Collectors.toList());
+ }
+
+ @Command
+ @NotifyChange({"positivePredictionResultForEvaluation", "negativePredictionResultForEvaluation"})
+ public void predictForEvaluation() {
+ positivePredictionResultForEvaluation = null;
+ negativePredictionResultForEvaluation = null;
+ Map<String, Integer> resultFromCoreModule =
+ evaluationService.predictForEvaluation(testTransactionSchema, testTransactiontable);
+ positivePredictionResultForEvaluation = resultFromCoreModule.get(PREDICTED_POSITIVE);
+ negativePredictionResultForEvaluation = resultFromCoreModule.get(PREDICTED_NEGATIVE);
+ }
+
+ @Command
+ public void uploadRealTransactionFromCsvFile(@BindingParam("param") File selectedFile) {
+ List<ObservedValue> observedValues = fileHandler.readObservedValueFromCsvFile(selectedFile);
+ coreClient.uploadToCoreModuleObservedValues(observedValues);
+ }
+
+ @Command
+ public void uploadRealTransactionFromDatabase() {
+ List<List<Object>> recordsInDatabase =
+ trainClient.getRecords(Map.of(SCHEMA_NAME, realTransactionSchema, TABLE_NAME, realTransactiontable));
+ String[] fieldNamesInArray = trainClient.getFieldNameInOrdinalPosition(
+ Map.of(SCHEMA_NAME, realTransactionSchema, TABLE_NAME, realTransactiontable));
+ List<String> fieldNames = Arrays.stream(fieldNamesInArray).collect(Collectors.toList());
+ int idIndex = fieldNames.indexOf(ID);
+ int transactionIdentifierIndex = fieldNames.indexOf(TRANSACTION_IDENTIFIER);
+ int fieldNameNumber = fieldNames.size();
+ List<ObservedValue> observedValues = new ArrayList<>();
+ for (List<Object> record : recordsInDatabase) {
+ ObservedValue observedValue = new ObservedValue(String.valueOf(record.get(transactionIdentifierIndex)),
+ (Integer) record.get(fieldNameNumber - 1));
+ observedValues.add(observedValue);
+ }
+ coreClient.uploadToCoreModuleObservedValues(observedValues);
+ Messagebox.show("Real transaction has been uploaded.");
+ }
+
+ @Command
+ @NotifyChange({"directoryProperties", "fileProperties", "fullPath"})
+ public void changeDirectory(@BindingParam("param") String selectedDirectoryName) {
+ File selectedDirectory = new File(selectedDirectoryName);
+ directoryProperties.clear();
+ directoryProperties.add(new DirectoryProperty("..", selectedDirectory.getParentFile()));
+ directoryProperties.addAll(getDirectoryProperties(selectedDirectory));
+ fileProperties = getFileProperties(selectedDirectory);
+ fullPath = selectedDirectoryName;
+ }
+
+ @Command
+ @NotifyChange("fileProperties")
+ public void displayFiles() {
+ System.out.println(selectedFolder);
+ }
+
+ @Command
+ @NotifyChange("confusionMatrixDto")
+ public void refreshSystemConfusionMatrix() {
+ confusionMatrixDto = coreClient.getSysTemConfusionMatrix();
+ }
+
+ @GlobalCommand
+ @NotifyChange("disabledTestPredictionStartButton")
+ public void enableTestPredictionStartButton() {
+ System.out.println("enable");
+ disabledTestPredictionStartButton = false;
+ }
+
+ @GlobalCommand
+ @NotifyChange("disabledTestPredictionStartButton")
+ public void disableTestPredictionStartButton() {
+ System.out.println("disable");
+ disabledTestPredictionStartButton = true;
+ }
+
+ @Command
+ @NotifyChange("confusionMatrixDto")
+ public void clearAllTransaction() {
+ coreClient.clearAllTransaction();
+ confusionMatrixDto = null;
+ }
+
+ private List<File> getDirectories(File file) {
+ return Arrays.stream(file.listFiles()).filter(e -> e.isDirectory() && e.isHidden() == false)
+ .collect(Collectors.toList());
+ }
+
+ private List<DirectoryProperty> getDirectoryProperties(File directory) {
+ return Arrays.stream(directory.listFiles()).filter(e -> e.isDirectory() && !e.isHidden())
+ .map(e -> new DirectoryProperty(e.getName(), new File(e.toURI()))).collect(Collectors.toList());
+ }
+
+ private List<FileProperty> getFileProperties(File directory) {
+ return Arrays.stream(directory.listFiles())
+ .filter(e -> e.isFile() && getExtensionByGuava(e.getName()).equals(CSV))
+ .map(e -> new FileProperty(e, e.getName())).collect(Collectors.toList());
+ }
+}
--- /dev/null
+restart.include.zklibs=/z[\\w]+-[\\w\\d-\.]+\.jar
--- /dev/null
+zk.homepage=start
+zk.zul-view-resolver-prefix=/zul
+zk.resource-uri=/zkres
+
--- /dev/null
+spring:
+ profiles:
+ default: "dev"
+ application:
+ name: CoreManager
+ datasource:
+ url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/common_fraud?serverTimezone=UTC
+ username: root
+ password: pwd
+logging:
+ level:
+ root: info
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /home/tomi/log/${spring.application.name}.log
+server:
+ port: 8082
+trainUrl: http://localhost:8085
+estimatorUrl: http://localhost:8083
+coreUrl: http://localhost:8081
+coreSoapUri: http://localhost:8081/ws/detection
+home:
+ directory: /home/tomi
+
+---
+spring:
+ config:
+ activate:
+ on-profile: dockerized
+# datasource:
+# url: ${DATASOURCE_URL}
+# username: ${MYSQL_USER}
+# password: ${MYSQL_PASSWORD}
+# jpa:
+# show-sql: true
+# generate-ddl: true
+# hibernate:
+# ddl-auto: create
+# formar_sql: true
+logging:
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /opt/app/log/${spring.application.name}.log
+server:
+ port: 8082
+trainUrl: http://train:8085
+estimatorUrl: http://estimator:8083
+coreUrl: http://core:8081
+coreSoapUri: http://core:8081/ws/detection
+
+---
\ No newline at end of file
--- /dev/null
+mvn jaxb2:xjc
\ No newline at end of file
--- /dev/null
+<language-addon>
+ <addon-name>zkspringboot-demo-addon</addon-name>
+ <version>1.0</version>
+ <language-name>xul/html</language-name>
+
+ <depends>zkmax</depends>
+
+ <stylesheet href="/css/static-globalstyles.css" type="text/css"/>
+ <stylesheet href="~./css/web-globalstyles.css" type="text/css"/>
+</language-addon>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<zk>
+ <config-name>zkspringboot-demo</config-name>
+ <library-property>
+ <name>org.zkoss.zul.nativebar</name>
+ <value>true</value>
+ </library-property>
+
+ <!--enable websocket support in ZK 8.5.1 (ZK-3799) -->
+ <!-- <listener>-->
+ <!-- <listener-class>org.zkoss.zkmax.au.websocket.WebSocketWebAppInit</listener-class>-->
+ <!-- </listener>-->
+
+ <!-- <richlet>-->
+ <!-- <richlet-name>ExampleRichlet</richlet-name><!– your preferred name –>-->
+ <!-- <richlet-class>org.zkoss.zkspringboot.demo.richlet.ExampleRichlet</richlet-class><!– your class name, of course –>-->
+ <!-- </richlet>-->
+ <!-- <richlet-mapping>-->
+ <!-- <richlet-name>ExampleRichlet</richlet-name>-->
+ <!-- <url-pattern>/richlet/example</url-pattern>-->
+ <!-- </richlet-mapping>-->
+</zk>
--- /dev/null
+.my-box2 {
+ border: 3px solid green;
+ padding: 15px;
+ display: block;
+ width: max-content;
+}
\ No newline at end of file
--- /dev/null
+.my-button {
+ color: Maroon;
+ font-weight: bold;
+}
\ No newline at end of file
--- /dev/null
+.bg {
+ background-image: url("plane.jpg");
+ background-size: cover;
+}
\ No newline at end of file
--- /dev/null
+.z-window-header{
+ /*background-color: aqua;*/
+ font-size: 40px;
+}
+
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?taglib uri="http://www.zkoss.org/dsp/web/core" prefix="c" ?>
+<zk>
+ <borderlayout viewModel="@id('vm') @init('hu.user.coremanager.viewmodel.EstimatorRegistryViewModel')"
+ vflex="min" width="100%">
+ <north vflex="min">
+ <listbox vflex="1" model="@load(vm.metricsAndTrainTaskNames)" mold="paging" pageSize="3"
+ sizedByContent="true" emptyMessage="There aren't any trained estimator">
+ <listhead>
+ <listheader label="SELECT" align="center"/>
+ <listheader label="Train Task name" align="center"/>
+ <listheader label="Estimator Id" align="center"/>
+ <listheader label="accuracy" align="center"/>
+ <listheader label="balanced accuracy" align="center"/>
+ <listheader label="sensitivity" align="center"/>
+ <listheader label="specifity" align="center"/>
+ <listheader label="prec" align="center"/>
+ <listheader label="recall" align="center"/>
+ <listheader label="PPV" align="center"/>
+ <listheader label="NPV" align="center"/>
+ <listheader label="FNR" align="center"/>
+ <listheader label="FPR" align="center"/>
+ <listheader label="FDR" align="center"/>
+ <listheader label="FOR" align="center"/>
+ <listheader label="f1" align="center"/>
+ <listheader label="f0.5" align="center"/>
+ <listheader label="f2" align="center"/>
+ <listheader label="MCC" align="center"/>
+ <listheader label="ROCAUC" align="center"/>
+ <listheader label="Youden" align="center"/>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell>
+ <button iconSclass="z-icon-plus"
+ onClick="@command('selectEstimatorId',param=each.metricsDto.estimatorId)"/>
+ </listcell>
+ <listcell label="@load(each.trainTaskName)"/>
+ <listcell label="@load(each.metricsDto.estimatorId)"/>
+ <listcell label="@load(each.metricsDto.accuracy)"/>
+ <listcell label="@load(each.metricsDto.balancedAccuracy)"/>
+ <listcell label="@load(each.metricsDto.sensitivity)"/>
+ <listcell label="@load(each.metricsDto.specifity)"/>
+ <listcell label="@load(each.metricsDto.prec)"/>
+ <listcell label="@load(each.metricsDto.recall)"/>
+ <listcell label="@load(each.metricsDto.ppv)"/>
+ <listcell label="@load(each.metricsDto.npv)"/>
+ <listcell label="@load(each.metricsDto.fnr)"/>
+ <listcell label="@load(each.metricsDto.fpr)"/>
+ <listcell label="@load(each.metricsDto.fdr)"/>
+ <listcell label="@load(each.metricsDto._for)"/>
+ <listcell label="@load(each.metricsDto.f1)"/>
+ <listcell label="@load(each.metricsDto.f05)"/>
+ <listcell label="@load(each.metricsDto.f2)"/>
+ <listcell label="@load(each.metricsDto.mcc)"/>
+ <listcell label="@load(each.metricsDto.rocauc)"/>
+ <listcell label="@load(each.metricsDto.youden)"/>
+ </listitem>
+ </template>
+ </listbox>
+ </north>
+ <east vflex="1" width="35%">
+ <grid model="@load(vm.estimatorRegistryParameters)" vflex="1" sizedByContent="true">
+ <auxhead>
+ <auxheader colspan="4" label="Registered Estimators" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1" label="Estimator Id" align="center"/>
+ <column hflex="1" label="Weight" align="center"/>
+ <column hflex="1" label="Modify" align="center"/>
+ <column hflex="1" label="Delete" align="center"/>
+ </columns>
+ <template name="model">
+ <row>
+ <label value="@load(each.estimatorId)"/>
+ <label value="@load(each.weight)"/>
+ <cell>
+ <hbox align="center" pack="center">
+ <button iconSclass="z-icon-plus"
+ onClick="@command('increaseWeight',param=each.estimatorId)"/>
+ <button iconSclass="z-icon-minus"
+ onClick="@command('decreaseWeight',param=each.estimatorId))"/>
+ </hbox>
+ </cell>
+ <button iconSclass="z-icon-trash-o"
+ onClick="@command('deleteEstimator',param=each.estimatorId)"/>
+ </row>
+ </template>
+ </grid>
+ </east>
+ <center>
+ <grid>
+ <auxhead>
+ <auxheader colspan="2" label="Register Estimator" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label hflex="1" value="Estimator Id" style="text-align: center"/>
+ </cell>
+ <cell>
+ <textbox hflex="1" value="@bind(vm.chosenEstimatorId)" style="text-align: center"/>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label hflex="1" value="Weight"/>
+ </cell>
+ <cell>
+ <spinner hflex="1"
+ constraint="no negative,no empty,no zero, min 1 max 10"
+ value="@bind(vm.chosenWeight)"
+ style="text-align: center"/>
+ </cell>
+ </row>
+ <row>
+ <cell/>
+ <cell>
+ <button hflex="1" label="REGISTER"
+ disabled="@load(vm.chosenEstimatorId eq null or vm.chosenWeight eq null)"
+ onClick="@command('registryEstimator')"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </center>
+ <west width="40%" vflex="1">
+ <grid>
+ <auxhead>
+ <auxheader colspan="3" label="Confusion Matrix" align="center"/>
+ </auxhead>
+
+ <columns>
+ <column hflex="1"></column>
+ <column hflex="1">Predicted positive</column>
+ <column hflex="1">Predicted negative</column>
+
+ </columns>
+ <rows>
+ <row align="center">
+ <cell style="background-color:#0093F9 ;color:white; border-left: 1px solid #0064ED; border-top: 1px solid #0064ED;">
+ <label>Observed positive</label>
+ </cell>
+ <cell>
+ <label value="@load(vm.truePositive)"/>
+ </cell>
+ <cell>
+ <label value="@load(vm.falseNegative)"/>
+ </cell>
+ </row>
+ <row align="center">
+ <cell style="background-color:#0093F9 ;color:white; border-left: 1px solid #0064ED; border-top: 1px solid #0064ED;">
+ <label>Observed negative</label>
+ </cell>
+ <cell>
+ <label value="@load(vm.falsePositive)"/>
+ </cell>
+ <cell>
+ <label value="@load(vm.trueNegative)"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </west>
+ </borderlayout>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?taglib uri="http://www.zkoss.org/dsp/web/core" prefix="c" ?>
+<zk>
+ <borderlayout height="400px" viewModel="@id('vm') @init('hu.user.coremanager.viewmodel.QualifyingViewModel')">
+ <west width="20%" splittable="true" collapsible="true">
+ <vbox>
+ <grid>
+ <auxhead>
+ <auxheader colspan="2" label="Test Prediction" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label value="Schema"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.schemas)" selectedItem="@bind(vm.testTransactionSchema)"
+ onChange="@command('changeTestTransactionSchema')" placeholder="choose schema"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label value="Table"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.testTransactionTables)" selectedItem="@bind(vm.testTransactiontable)"
+ placeholder="choose table" style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label value="Prediction"/>
+ </cell>
+ <cell>
+ <button disabled="@load(vm.disabledTestPredictionStartButton or empty vm.testTransactiontable)" label="START" onClick="@command('predictForEvaluation')"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ <grid visible="@load(not empty vm.positivePredictionResultForEvaluation)">
+ <auxhead>
+ <auxheader colspan="2" label="Test result" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label value="Positive"/>
+ </cell>
+ <textbox hflex="1" value="@load(vm.positivePredictionResultForEvaluation)" readonly="true"
+ placeholder="predicted positive" style="text-align:center"/>
+ </row>
+ <row>
+ <cell>
+ <label value="Negative"/>
+ </cell>
+ <cell>
+ <textbox hflex="1" value="@load(vm.negativePredictionResultForEvaluation)"
+ readonly="true" placeholder="predicted negative" style="text-align:center"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </vbox>
+
+ </west>
+ <east autoscroll="true" width="40%" splittable="true" collapsible="true">
+ <grid>
+ <auxhead>
+ <auxheader colspan="3" label="PREDICTION QUALITY" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"></column>
+ <column hflex="1">Predicted positive</column>
+ <column hflex="1">Predicted negative</column>
+ </columns>
+ <rows>
+ <row>
+ <cell style="background-color:#0093F9 ;color:white; border-left: 1px solid #0064ED; border-top: 1px solid #0064ED;">
+ <label style="color:#FFFFFF;background-color:#0093F9">Observed positive</label>
+ </cell>
+ <cell align="center">
+ <label visible="@load(vm.confusionMatrixDto ne null)" value="@load(vm.confusionMatrixDto.truePositive)"/>
+ </cell>
+ <cell align="center">
+ <label visible="@load(vm.confusionMatrixDto ne null)" value="@load(vm.confusionMatrixDto.falseNegative)"/>
+ </cell>
+ </row>
+ <row>
+ <cell style="background-color:#0093F9 ;color:white; border-left: 1px solid #0064ED; border-top: 1px solid #0064ED;">
+ <label>Observed negative</label>
+ </cell>
+ <cell align="center">
+ <label visible="@load(vm.confusionMatrixDto ne null)" value="@load(vm.confusionMatrixDto.falsePositive)"/>
+ </cell>
+ <cell align="center">
+ <label visible="@load(vm.confusionMatrixDto ne null)" value="@load(vm.confusionMatrixDto.trueNegative)"/>
+ </cell>
+ </row>
+ <row>
+ <cell/>
+ <cell>
+ </cell>
+ <cell>
+ <button hflex="1" iconSclass="z-icon-trash" onClick="@command('clearAllTransaction')"/>
+ <button hflex="1" iconSclass="z-icon-refresh" onClick="@command('refreshSystemConfusionMatrix')"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </east>
+ <center autoscroll="true">
+ <tabbox vflex="1" mold="accordion">
+ <tabs>
+ <tab label="Upload real transaction from CSV file"/>
+ <tab label="Upload real transaction from database"/>
+ </tabs>
+ <tabpanels>
+ <tabpanel>
+ <borderlayout vflex="1">
+ <north>
+ <grid>
+ <auxhead>
+ <auxheader>
+ <hbox align="center">
+<!-- <label hflex="1" value="CSV File upload"/>-->
+ <hbox hflex="1" align="center">
+ <label value="Path: "/>
+ <textbox hflex="min" value="@load(vm.fullPath)"/>
+ </hbox>
+
+ </hbox>
+
+ </auxheader>
+ </auxhead>
+
+ <columns>
+ <column hflex="1"/>
+ </columns>
+ </grid>
+ </north>
+ <west hflex="1">
+ <listbox model="@load(vm.directoryProperties)" vflex="true">
+ <listhead>
+ <listheader label="Directory"/>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.getDisplayName())"
+ onClick="@command('changeDirectory',param=each.directory)"/>
+ </listitem>
+ </template>
+ </listbox>
+ </west>
+ <center hflex="1">
+ <listbox model="@load(vm.fileProperties)" mold="paging" pageSize="4" vflex="true"
+ selectedIndex="@bind(vm.selectedFileNameIndex)">
+ <listhead>
+ <listheader label="File name" align="center"/>
+ <listheader label="UPLOAD" align="center"/>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.fileName)"/>
+ <listcell>
+ <button iconSclass="z-icon-plus"
+ onClick="@command('uploadRealTransactionFromCsvFile',param=each.file)"/>
+ </listcell>
+ </listitem>
+ </template>
+ </listbox>
+ </center>
+ </borderlayout>
+ </tabpanel>
+ <tabpanel>
+ <borderlayout vflex="1">
+ <north>
+ <hbox align="center" pack="center">
+ <label hflex="1" value="Schema"/>
+ <combobox hflex="2" model="@load(vm.schemas)" selectedItem="@bind(vm.realTransactionSchema)"
+ onChange="@command('changeRealTransactionSchema')" placeholder="choose schema"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ <label hflex="1" value="Table" style="text-align: center"/>
+ <combobox hflex="2" model="@load(vm.realTransactionTables)" selectedItem="@bind(vm.realTransactiontable)"
+ onChange="@command('changeRealTransactionTable')" placeholder="choose table"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ <button disabled="@load(empty vm.realTransactiontable)" iconSclass="z-icon-upload" tooltiptext="Upload database content" onClick="@command('uploadRealTransactionFromDatabase')"/>
+ </hbox>
+ </north>
+ <center autoscroll="true">
+ <grid model="@load(vm.fieldProperties)" emptyMessage="Please select a database" style="text-align:center">
+ <columns>
+ <column hflex="1" label="Field name" align="center"/>
+ <column hflex="1" label="Field type" align="center"/>
+ </columns>
+ <template name="model">
+ <row>
+ <label value="@load(each.fieldName)"/>
+ <label value="@load(each.fieldType)"/>
+ </row>
+ </template>
+ </grid>
+ </center>
+ </borderlayout>
+ </tabpanel>
+ </tabpanels>
+ </tabbox>
+ </center>
+ </borderlayout>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?taglib uri="http://www.zkoss.org/dsp/web/core" prefix="c" ?>
+<?page title="Core Manager"?>
+<zk>
+<!-- <style src="~./css/style.css"/>-->
+ <window border="normal" height="100%" width="100%">
+ <caption>
+ <label value="Core Manager"/>
+ </caption>
+ <tabbox vflex="1">
+ <tabs>
+ <tab label="Estimator Registry"/>
+ <tab label="System Qualifying"/>
+ </tabs>
+ <tabpanels>
+ <tabpanel style="overflow: auto">
+ <include src="~./zul/estimatorRegistry.zul"/>
+ </tabpanel>
+ <tabpanel>
+ <include src="~./zul/qualifying.zul"/>
+ </tabpanel>
+ </tabpanels>
+ </tabbox>
+ </window>
+</zk>
\ No newline at end of file
--- /dev/null
+<wsdl:definitions xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/" xmlns:sch="http://spring.io/guides/gs-producing-web-service" xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/" xmlns:tns="http://spring.io/guides/gs-producing-web-service" targetNamespace="http://spring.io/guides/gs-producing-web-service">
+ <wsdl:types>
+ <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://spring.io/guides/gs-producing-web-service">
+ <xs:element name="generateDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="detection_request" type="tns:detection_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="generateGenericDetectionRequest">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="unbounded" minOccurs="1" name="detection_request" type="tns:generic_detection_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="storeRealTransactionRequest">
+ <xs:complexType>
+ <xs:sequence maxOccurs="unbounded" minOccurs="1">
+ <xs:element maxOccurs="unbounded" minOccurs="1" name="store_request" type="tns:store_request"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="storeRealTransactionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="store_result" type="tns:store_result"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="generateDetectionResponse">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="detection_response" type="tns:detection_response"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ <xs:complexType name="detection_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="card_number" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_type" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="timestamp" type="xs:dateTime"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="amount" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="currency_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="response_code" type="xs:int"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="country_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="vendor_code" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="generic_detection_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="field_name" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="value" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="detection_response">
+ <xs:sequence>
+ <xs:element name="prediction" type="xs:int"/>
+ <xs:element name="negative_probability" type="xs:double"/>
+ <xs:element name="positive_probability" type="xs:double"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="store_request">
+ <xs:sequence>
+ <xs:element maxOccurs="1" minOccurs="1" name="transaction_id" type="xs:string"/>
+ <xs:element maxOccurs="1" minOccurs="1" name="fraud" type="xs:int"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="store_result">
+ <xs:sequence>
+ <xs:element name="result" type="xs:string"/>
+ <xs:element name="record_number" type="xs:int"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:schema>
+ </wsdl:types>
+ <wsdl:message name="storeRealTransactionResponse">
+ <wsdl:part element="tns:storeRealTransactionResponse" name="storeRealTransactionResponse"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="storeRealTransactionRequest">
+ <wsdl:part element="tns:storeRealTransactionRequest" name="storeRealTransactionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateDetectionRequest">
+ <wsdl:part element="tns:generateDetectionRequest" name="generateDetectionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateDetectionResponse">
+ <wsdl:part element="tns:generateDetectionResponse" name="generateDetectionResponse"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:message name="generateGenericDetectionRequest">
+ <wsdl:part element="tns:generateGenericDetectionRequest" name="generateGenericDetectionRequest"> </wsdl:part>
+ </wsdl:message>
+ <wsdl:portType name="DetectionPort">
+ <wsdl:operation name="storeRealTransaction">
+ <wsdl:input message="tns:storeRealTransactionRequest" name="storeRealTransactionRequest"> </wsdl:input>
+ <wsdl:output message="tns:storeRealTransactionResponse" name="storeRealTransactionResponse"> </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateDetection">
+ <wsdl:input message="tns:generateDetectionRequest" name="generateDetectionRequest"> </wsdl:input>
+ <wsdl:output message="tns:generateDetectionResponse" name="generateDetectionResponse"> </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateGenericDetection">
+ <wsdl:input message="tns:generateGenericDetectionRequest" name="generateGenericDetectionRequest"> </wsdl:input>
+ </wsdl:operation>
+ </wsdl:portType>
+ <wsdl:binding name="DetectionPortSoap11" type="tns:DetectionPort">
+ <soap:binding style="document" transport="http://schemas.xmlsoap.org/soap/http"/>
+ <wsdl:operation name="storeRealTransaction">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="storeRealTransactionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ <wsdl:output name="storeRealTransactionResponse">
+ <soap:body use="literal"/>
+ </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateDetection">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="generateDetectionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ <wsdl:output name="generateDetectionResponse">
+ <soap:body use="literal"/>
+ </wsdl:output>
+ </wsdl:operation>
+ <wsdl:operation name="generateGenericDetection">
+ <soap:operation soapAction=""/>
+ <wsdl:input name="generateGenericDetectionRequest">
+ <soap:body use="literal"/>
+ </wsdl:input>
+ </wsdl:operation>
+ </wsdl:binding>
+ <wsdl:service name="DetectionPortService">
+ <wsdl:port binding="tns:DetectionPortSoap11" name="DetectionPortSoap11">
+ <soap:address location="http://localhost:8081/ws"/>
+ </wsdl:port>
+ </wsdl:service>
+</wsdl:definitions>
\ No newline at end of file
--- /dev/null
+package hu.user.coremanager;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class CoreManagerApplicationTests {
+
+ @Test
+ void contextLoads() {
+ }
+
+}
--- /dev/null
+# set base image (host OS)
+FROM python:3.8
+
+# set the working directory in the container
+WORKDIR /code
+
+# copy the dependencies file to the working directory
+COPY requirements.txt .
+
+# install dependencies
+RUN pip install -r requirements.txt
+
+# copy the content of the local src directory to the working directory
+COPY src/ .
+
+# command to run on container start
+CMD [ "python", "./server.py" ]
\ No newline at end of file
--- /dev/null
+click==8.1.3
+Flask==2.0.3
+importlib-metadata==4.12.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+joblib==1.2.0
+lightgbm==3.3.3
+MarkupSafe==2.1.1
+mysql-connector==2.2.9
+mysql-connector-python==8.0.28
+numpy==1.21.6
+pandas==1.3.5
+pika==1.2.1
+python-dateutil==2.8.2
+pytz==2022.2.1
+scikit-learn==1.0.2
+scipy==1.7.3
+six==1.16.0
+threadpoolctl==3.1.0
+typing_extensions==4.3.0
+Werkzeug==2.2.2
+xgboost==1.6.2
+zipp==3.8.1
--- /dev/null
+import json
+import logging
+import mysql.connector
+from mysql.connector import pooling
+import numpy as np
+import pickle
+import sklearn
+import lightgbm
+
+
+class NumpyMySQLConverter(mysql.connector.conversion.MySQLConverter):
+ """ A mysql.connector Converter that handles Numpy types """
+
+ def _float32_to_mysql(self, value):
+ return float(value)
+
+ def _float64_to_mysql(self, value):
+ return float(value)
+
+ def _int32_to_mysql(self, value):
+ return int(value)
+
+ def _int64_to_mysql(self, value):
+ return int(value)
+
+
+class Handler:
+ def __init__(self, database_url, database_user, database_password):
+ self.logger = logging.getLogger("estimator.server.database")
+ self.database_url = database_url
+ self.database_user = database_user
+ self.database_password = database_password
+ # self.connection = self.get_connection(self.database_url, self.database_user, self.database_password)
+ self.connection_pool = self.get_connection_pool(self.database_url, self.database_user, self.database_password)
+
+ def get_connection_pool(self, database_url, database_user_name, database_password):
+ try:
+ connection_pool = pooling.MySQLConnectionPool(pool_name="estimator_pool",
+ pool_size=8,
+ pool_reset_session=True,
+ host=database_url,
+ user=database_user_name,
+ password=database_password)
+ self.logger.debug("database connection pool created in train modul")
+ return connection_pool
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL connection error: {err.msg}")
+
+ def get_connection_from_pool(self):
+ try:
+ connection=self.connection_pool.get_connection()
+ connection.set_converter_class(NumpyMySQLConverter)
+ return connection
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ # def get_connection(self, database_url, database_user_name, database_password):
+ # try:
+ # connection = mysql.connector.connect(
+ # # pool_name="local",
+ # # pool_size=16,
+ # host=database_url,
+ # user=database_user_name,
+ # password=database_password)
+ # self.logger.debug("database connection created in estimator modul")
+ # connection.set_converter_class(NumpyMySQLConverter)
+ # return connection
+ # except mysql.connector.Error as err:
+ # self.logger.error(f"MySQL connection error: {err.msg}")
+
+ def get_schemas(self):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SHOW SCHEMAS"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = self.build_result_from_query_result(query_result)
+ return result if query_result is not None else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_tables_of_common_fraud(self, schema_name):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT distinct TABLE_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}'"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = self.build_result_from_query_result(query_result)
+ return result if query_result is not None else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def build_result_from_query_result(self, query_result):
+ result = list()
+ for item in query_result:
+ temp = item[0]
+ result.append(temp)
+ return result
+
+ def get_available_estimator_ids(self):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT id FROM common_fraud.estimator"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ response = list()
+ if query_result is not None:
+ for item in query_result:
+ response.append(item[0])
+ return response if query_result is not None else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_estimator_properties(self, id):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT * FROM common_fraud.estimator WHERE id = %s"
+ try:
+ parameter = (id,)
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ train_task_id = query_result[1]
+ estimator_properties = pickle.loads(query_result[3])
+ pipeline = estimator_properties.get("pipeline")
+ label_encoder_registry = estimator_properties.get("label_encoder_registry")
+ encoding_parameters = estimator_properties.get("encoding_parameters")
+ feature_engineering_parameters = estimator_properties.get("feature_engineering_parameters")
+ encoded_field_names = estimator_properties.get("encoded_field_names")
+ time_base_field_name = estimator_properties.get("time_base_field_name")
+ schema_name = estimator_properties.get("schema_name")
+ feature_engineered_table_name = estimator_properties.get("feature_engineered_table_name")
+ detailed_information_about_table = estimator_properties.get("detailed_information_about_table")
+ feature_engineered_insert_script = estimator_properties.get("feature_engineered_insert_script")
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"estimator object loaded from database, estimator_id: {id}")
+ return train_task_id, pipeline, label_encoder_registry, encoding_parameters, feature_engineering_parameters, encoded_field_names, time_base_field_name, schema_name, feature_engineered_table_name, detailed_information_about_table, feature_engineered_insert_script if query_result is not None else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def load_label_encoder_object(self, id):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT * FROM common_fraud.label_encoder WHERE id = %s"
+ try:
+ parameter = (id,)
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ result = None
+ if query_result is not None:
+ result = pickle.loads(query_result[1])
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_feature_to_be_engineered_and_time_base_field_value_for_feature_engineering(self, schema_name,
+ feature_engineered_table_name,
+ feature_to_be_engineered,
+ time_base_field_name,
+ # time_of_input_data,
+ retrospective_time,
+ card_number_value):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ float_retrespective_time = float(retrospective_time)
+ sql_select_query = f"SELECT {feature_to_be_engineered}, {time_base_field_name} FROM {schema_name}.{feature_engineered_table_name} WHERE card_number = %s and {time_base_field_name} > %s ORDER BY {time_base_field_name} DESC"
+ try:
+ parameter = (card_number_value, float_retrespective_time)
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ return np.array(query_result) if query_result is not None else np.array([[0,0]])
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_feature_engineered_record(self, insert_script, data):
+ connection_object = self.get_connection_from_pool()
+ cursor = connection_object.cursor()
+ try:
+ cursor.execute(insert_script, data)
+ connection_object.commit()
+ connection_object.close()
+ self.logger.debug("record persisted in feature engineered table")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
--- /dev/null
+import logging
+import math
+import statistics
+import multiprocessing as mp
+from multiprocessing import Process, Pool
+import numpy as np
+
+CARD_NUMBER = "card_number"
+UNDER_SCORE = "_"
+
+
+class Engineer:
+ def __init__(self, time_base_field_name, encoded_field_names, feature_engineering_parameters):
+ self.logger = logging.getLogger("estimator.server.database.engineer")
+ self.time_base_field_name = time_base_field_name
+ self.encoded_field_names = encoded_field_names
+ self.feature_engineering_parameters = feature_engineering_parameters
+ self.chosen_feature_field_name = self.feature_engineering_parameters.get("chosen_feature_field")
+ self.intervals = self.feature_engineering_parameters.get("intervals")
+
+ def process(self, encoded_input_data, previous_feature_and_time_base_array_by_intervals,
+ index_of_feature_to_be_engineered, index_of_time_base_field):
+ current_feature_value_to_be_engineered = encoded_input_data[index_of_feature_to_be_engineered]
+ current_time_stamp_value = encoded_input_data[index_of_time_base_field]
+ for interval, previous_feature_and_time_base_array in previous_feature_and_time_base_array_by_intervals.items():
+ previous_feature_array = previous_feature_and_time_base_array[:, :1]
+ previous_time_stamp_array = previous_feature_and_time_base_array[:, 1:]
+ previous_feature_list = previous_feature_array.flatten().tolist()
+ previous_time_stamp_list = previous_time_stamp_array.flatten().tolist()
+ combined_feature_list = list()
+ combined_time_stamp_list = list()
+ combined_feature_list.append(current_feature_value_to_be_engineered)
+ combined_feature_list.extend(previous_feature_list)
+ combined_time_stamp_list.append(current_time_stamp_value)
+ combined_time_stamp_list.extend(previous_time_stamp_list)
+ generated_features_based_on_chosen_feature = self.get_generated_features_based_on_chosen_feature(
+ current_feature_value_to_be_engineered, combined_feature_list)
+ generated_features_based_on_transaction_number = self.get_generated_features_based_on_transaction_number(
+ current_time_stamp_value, combined_time_stamp_list, interval)
+ generated_features_based_on_transaction_interval = self.get_generated_features_based_on_transaction_interval(
+ combined_time_stamp_list)
+ encoded_input_data.extend(generated_features_based_on_chosen_feature)
+ encoded_input_data.extend(generated_features_based_on_transaction_number)
+ encoded_input_data.extend(generated_features_based_on_transaction_interval)
+ return encoded_input_data
+
+ def get_generated_features_based_on_chosen_feature(self, current_feature_value,
+ combined_feature_list):
+ average_feature_value = statistics.mean(combined_feature_list)
+ deviation_feature_value = statistics.stdev(combined_feature_list)
+ median_feature_value = statistics.median(combined_feature_list)
+ result = list()
+ feature_per_average_feature_value = current_feature_value / average_feature_value if average_feature_value != 0 else 0
+ result.append(feature_per_average_feature_value)
+ feature_minus_average_feature_value = current_feature_value - average_feature_value
+ result.append(feature_minus_average_feature_value)
+ feature_per_median_feature_value = current_feature_value / median_feature_value if median_feature_value != 0 else 0
+ result.append(feature_per_median_feature_value)
+ feature_minus_median_feature_value = current_feature_value - median_feature_value
+ result.append(feature_minus_median_feature_value)
+ feature_minus_average_feature_per_deviation_feature_value = (current_feature_value - average_feature_value) / deviation_feature_value if deviation_feature_value != 0 else 0
+ result.append(feature_minus_average_feature_per_deviation_feature_value)
+ feature_minus_average_feature_minus_deviation_feature_value = current_feature_value - average_feature_value - deviation_feature_value
+ result.append(feature_minus_average_feature_minus_deviation_feature_value)
+ return result
+
+ def get_generated_features_based_on_transaction_number(self, current_time_stamp_value, combined_time_stamp_list,
+ interval):
+ transaction_list_on_last_day = list()
+ for time_stamp in combined_time_stamp_list:
+ if time_stamp > current_time_stamp_value - 1:
+ transaction_list_on_last_day.append(time_stamp)
+ transaction_number_on_last_day = len(transaction_list_on_last_day)
+ average_transaction_number = len(combined_time_stamp_list) / interval
+ median_transaction_number = self.get_median_transaction_number(combined_time_stamp_list)
+ result = list()
+ transaction_number_on_current_day_per_daily_average_transaction_number = transaction_number_on_last_day / average_transaction_number if average_transaction_number != 0 else 0
+ result.append(transaction_number_on_current_day_per_daily_average_transaction_number)
+ transaction_number_on_current_day_minus_daily_average_transaction_number = transaction_number_on_last_day - average_transaction_number
+ result.append(transaction_number_on_current_day_minus_daily_average_transaction_number)
+ transaction_number_on_current_day_per_daily_median_transaction_number = transaction_number_on_last_day / median_transaction_number if median_transaction_number != 0 else 0
+ result.append(transaction_number_on_current_day_per_daily_median_transaction_number)
+ transaction_number_on_current_day_minus_daily_median_transaction_number = transaction_number_on_last_day - median_transaction_number
+ result.append(transaction_number_on_current_day_minus_daily_median_transaction_number)
+ return result
+
+ def get_median_transaction_number(self,combined_time_stamp_list):
+ time_stamp_by_date_in_julian_format=dict()
+ for time_stamp_date_in_julian_format in combined_time_stamp_list:
+ current_day_in_julian_format=int(time_stamp_date_in_julian_format)
+ if time_stamp_by_date_in_julian_format.get(current_day_in_julian_format) is None:
+ time_stamps_on_given_day=list()
+ time_stamps_on_given_day.append(time_stamp_date_in_julian_format)
+ time_stamp_by_date_in_julian_format[current_day_in_julian_format]=time_stamps_on_given_day
+ else:
+ time_stamps_on_given_day=time_stamp_by_date_in_julian_format.get(current_day_in_julian_format)
+ time_stamps_on_given_day.append(time_stamp_date_in_julian_format)
+ transaction_numbers_on_days=list()
+ for time_stamps in time_stamp_by_date_in_julian_format.values():
+ transaction_numbers_on_days.append(len(time_stamps))
+ median_transsction_number=statistics.median(transaction_numbers_on_days)
+ return median_transsction_number
+
+ def get_generated_features_based_on_transaction_interval(self, combined_time_stamp_list):
+ number_of_time_stamps = len(combined_time_stamp_list)
+ if number_of_time_stamps>=2:
+ current_distance = combined_time_stamp_list[0] - combined_time_stamp_list[1]
+ distances = list()
+ for i in range(number_of_time_stamps - 1):
+ distance = combined_time_stamp_list[i] - combined_time_stamp_list[i + 1]
+ distances.append(distance)
+
+ average_distance_between_transactions = statistics.mean(distances)
+ median_distance_between_transactions = statistics.median(distances)
+ result = list()
+ average_distance_per_current_distance = average_distance_between_transactions / current_distance if current_distance != 0 else 0
+ average_distance_time_minus_current_distance_between_transactions = average_distance_between_transactions - current_distance
+ median_distance_per_current_distance_between_transactions = median_distance_between_transactions / current_distance if current_distance != 0 else 0
+ median_distance_minus_current_distance_between_transactions = median_distance_between_transactions - current_distance
+ else:
+ average_distance_per_current_distance = 0
+ average_distance_time_minus_current_distance_between_transactions = 0
+ median_distance_per_current_distance_between_transactions = 0
+ median_distance_minus_current_distance_between_transactions = 0
+
+ result.append(average_distance_per_current_distance)
+ result.append(average_distance_time_minus_current_distance_between_transactions)
+ result.append(median_distance_per_current_distance_between_transactions)
+ result.append(median_distance_minus_current_distance_between_transactions)
+ return result
--- /dev/null
+import sys
+from datetime import datetime
+import dateutil.parser
+import time
+import logging
+import os
+
+import numpy as np
+import pandas as pd
+import pickle
+from xgboost import XGBClassifier
+from flask import Flask, jsonify, request, Response
+
+import database_module
+import engineering_module
+
+FRAUD_TYPE = "fraud_type"
+ID = "id"
+TYPE = "type"
+NAME = "name"
+INT = "int"
+FLOAT = "float"
+STRING = "str"
+DATETIME = "datetime"
+LABEL_ENCODER = "label_encoder"
+JULIAN = "julian"
+INTERVALS = "intervals"
+CARD_NUMBER = "card_number"
+CHOSEN_FEATURE_FIELD_FOR_FEATURE_ENGINEERING = "chosen_feature_field"
+COMMON_FRAUD_SCHEMA = "common_fraud"
+
+server = Flask(__name__)
+
+
+class InValidRequestParameterException(Exception):
+ pass
+
+
+class NotImplementedConversionTypeException(Exception):
+ pass
+
+
+class MissingInputFieldNameException(Exception):
+ pass
+
+
+@server.route('/prediction', methods=['POST'])
+def get_prediction():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ try:
+ start_predict = time.time()
+ result = estimator_holder.predict_using_all_estimator(request_parameter)
+ finish_predict = time.time()
+ predict_processing_time = finish_predict - start_predict
+ logger.info(f"prediction finished, result: {result}, processing time: {predict_processing_time}")
+ return jsonify(result)
+ except MissingInputFieldNameException as e:
+ details = e.args[0]
+ message = details.get("message")
+ return Response(message, status=400)
+ except InValidRequestParameterException as e:
+ details = e.args[0]
+ message = details.get("message")
+ return Response(message, status=400)
+ except NotImplementedConversionTypeException as e:
+ details = e.args[0]
+ message = details.get("message")
+ return Response(message, status=400)
+
+@server.route('/estimator', methods=['POST'])
+def add_estimator():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ id = request_parameter.get("estimator_id")
+ registered_estimator_ids = estimator_holder.get_all_registered_estimator_id()
+ if id in registered_estimator_ids:
+ message = f"estimator with this id is already registered, id: {id}"
+ logger.error(message)
+ return Response(message, status=400)
+ try:
+ estimator_ids = estimator_holder.register_estimator(id)
+ logger.info(f"estimator (id: {id}) registered")
+ return jsonify(estimator_ids)
+ except InValidRequestParameterException as e:
+ details = e.args[0]
+ message = details.get("message")
+ return Response(message, status=400)
+
+
+@server.route('/estimator', methods=['GET'])
+def get_estimator():
+ result = estimator_holder.get_all_registered_estimator_id()
+ return jsonify(result)
+
+
+@server.route('/estimator/<id>', methods=['DELETE'])
+def delete_estimator(id):
+ try:
+ estimator_ids = estimator_holder.delete_estimator(id)
+ logger.info(f"estimator (id: {id}) deleted")
+ return jsonify(estimator_ids)
+ except InValidRequestParameterException as e:
+ details = e.args[0]
+ message = details.get("message")
+ logger.error(message)
+ return Response(message, status=400)
+
+
+class EstimatorHolder:
+ def __init__(self):
+ self.estimator_by_id = dict()
+
+ def register_estimator(self, id):
+ available_estimator_ids = database_handler.get_available_estimator_ids()
+ if id not in available_estimator_ids:
+ message = f"estimator doesn't exist, estimator id: {id}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ estimator = Estimator(id)
+ estimator.initialize()
+ self.estimator_by_id[id] = estimator
+ return self.get_all_registered_estimator_id()
+
+ def delete_estimator(self, id):
+ try:
+ self.estimator_by_id.pop(int(id))
+ except KeyError:
+ message = f"estimator isn't registered with this id, estimator id: {id}"
+ logger.error(message)
+ raise InValidRequestParameterException(({"message": message}))
+ return self.get_all_registered_estimator_id()
+
+ def get_all_registered_estimator_id(self):
+ result = list()
+ for item in self.estimator_by_id.keys():
+ result.append(item)
+ return result
+
+ def predict_using_all_estimator(self, raw_input_data):
+ prediction_by_id = dict()
+ for id, estimator in self.estimator_by_id.items():
+ prediction, positive_probability, negative_probability = estimator.data_preparation_and_predict(
+ raw_input_data)
+ prediction_by_id[id] = {
+ "prediction": prediction,
+ "positive_probability": positive_probability,
+ "negative_probability": negative_probability
+ }
+ return prediction_by_id
+
+
+class Estimator:
+ def __init__(self, id):
+ self.id = id
+ self.train_task_id = None
+ self.pipeline = None
+ self.label_encoder_by_field_name = dict()
+ self.encoding_parameters = dict()
+ self.feature_engineering_parameters = dict()
+ self.encoded_field_names = None
+ self.time_base_field_name = None
+ self.schema_name = None
+ self.feature_engineered_table_name = None
+ self.detailed_information_about_table = None
+ self.feature_engineered_insert_script = None
+
+ def initialize(self):
+ train_task_id, pipeline, label_encoder_registry, encoding_parameters, feature_engineering_parameters, encoded_field_names, time_base_field_name, schema_name, feature_engineered_table_name, detailed_information_about_table, feature_engineered_insert_script = database_handler.get_estimator_properties(
+ self.id)
+ self.train_task_id = train_task_id
+ self.pipeline = pipeline
+ for field_name, label_encoder_id in label_encoder_registry.items():
+ label_encoder = database_handler.load_label_encoder_object(label_encoder_id)
+ self.label_encoder_by_field_name[field_name] = label_encoder
+ self.encoding_parameters = encoding_parameters
+ self.feature_engineering_parameters = feature_engineering_parameters
+ self.encoded_field_names = self.remove_fraud_type_field_name(encoded_field_names)
+ self.time_base_field_name = time_base_field_name
+ self.schema_name = schema_name
+ self.feature_engineered_table_name = feature_engineered_table_name
+ self.detailed_information_about_table = detailed_information_about_table
+ self.feature_engineered_insert_script = feature_engineered_insert_script
+
+ def remove_fraud_type_field_name(self, field_names):
+ for field_name, encoding_type in self.encoding_parameters.items():
+ if encoding_type == FRAUD_TYPE:
+ field_names.remove(field_name)
+ return field_names
+
+ def data_preparation_and_predict(self, raw_input_data):
+ input_data = self.data_preprocess(raw_input_data)
+ data_set = self.encode(input_data)
+ if self.feature_engineering_parameters is not None:
+ data_set = self.feature_engineering(data_set)
+ prediction, positive_probability, negative_probability = self.predict(data_set)
+ return prediction, positive_probability, negative_probability
+
+ def data_preprocess(self, raw_input_data):
+ preprocessed_inpud_data = dict()
+ field_names_in_input_data = list(raw_input_data.keys())
+ required_input_field_names = self.get_required_input_field_names()
+ is_input_contains_all_required_field_names = self.check_if_input_contains_required_field_names(
+ required_input_field_names, field_names_in_input_data)
+ if not is_input_contains_all_required_field_names:
+ message = "There is a missing input field name in the request"
+ logger.error(message)
+ raise MissingInputFieldNameException(({"message": message}))
+ required_input_field_name_by_type = self.get_required_input_field_name_by_type()
+ for required_input_field_name, required_type_in_python in required_input_field_name_by_type.items():
+ current_field_value = raw_input_data[required_input_field_name]
+ preprocessed_inpud_data[required_input_field_name] = self.convert_to_own_type(required_input_field_name,
+ current_field_value,
+ required_type_in_python)
+ return preprocessed_inpud_data
+
+ def check_if_input_contains_required_field_names(self, required_field_names, input_field_names):
+ return all(i in input_field_names for i in required_field_names)
+
+ def get_required_input_field_names(self):
+ return list(self.get_required_input_field_name_by_type().keys())
+
+ def get_required_input_field_name_by_type(self):
+ field_properties = self.detailed_information_about_table.get("fields")
+ chosen_fraud_type_field_name = None
+ for field_name_in_encoding_parameters, planned_encoding_type in self.encoding_parameters.items():
+ if planned_encoding_type == FRAUD_TYPE:
+ chosen_fraud_type_field_name = field_name_in_encoding_parameters
+ field_names_without_id_and_fraud_type_by_field_type = dict()
+ for field_property in field_properties:
+ field_name = field_property.get(NAME)
+ field_type_in_database = field_property.get(TYPE)
+ if field_name != ID and field_name != chosen_fraud_type_field_name:
+ field_names_without_id_and_fraud_type_by_field_type[field_name] = self.convert_to_python_data_type(
+ field_type_in_database)
+ return field_names_without_id_and_fraud_type_by_field_type
+
+ def convert_to_python_data_type(self, field_type_in_database):
+ database_python_data_type_mapping = {
+ "int": "int",
+ "float": "float",
+ "varchar": "str",
+ "datetime": "datetime"
+ }
+ data_type_in_python = database_python_data_type_mapping.get(field_type_in_database)
+ return data_type_in_python
+
+ def convert_to_own_type(self, current_field_name, current_field_value, required_type):
+ if required_type == INT:
+ try:
+ converted_value = int(current_field_value)
+ except ValueError as e:
+ message = f"can not convert raw input data to integer , field name: {current_field_name}, raw data: {current_field_value}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif required_type == FLOAT:
+ try:
+ converted_value = float(current_field_value)
+ except ValueError as e:
+ message = f"can not convert raw input data to float, field name: {current_field_name}, raw data: {current_field_value}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif required_type == STRING:
+ try:
+ converted_value = str(current_field_value)
+ except ValueError as e:
+ message = f"can not convert raw input data to string, field name: {current_field_name}, raw data: {current_field_value}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif required_type == DATETIME:
+ converted_value = self.convert_rfc3339_compliant(current_field_value)
+ if converted_value is None:
+ converted_value = self.convert_iso8601_compliant(current_field_value)
+ if converted_value is None:
+ message = f"can not convert raw input data to datetime, field name: {current_field_name}, raw data: {current_field_value}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ else:
+ message = f"unknow conversion type in preprocessing step, type: {required_type}"
+ logger.error(message)
+ raise NotImplementedConversionTypeException({{"message": message}})
+ return converted_value
+
+ def convert_rfc3339_compliant(self, input):
+ converted_value = None
+ try:
+ converted_value = datetime.fromisoformat(input)
+ except ValueError:
+ logger.debug(f"Not RFC 3339 convertable timestamp, value{input}")
+ return converted_value
+
+ def convert_iso8601_compliant(self, input):
+ converted_value = None
+ try:
+ converted_value = dateutil.parser.isoparse(input)
+ except ValueError:
+ logger.debug(f"Not ISO 8601 convertable timestamp, value{input}")
+ return converted_value
+
+ def encode(self, input_data):
+ encoded_field_values = list()
+ for field_name in self.encoded_field_names:
+ current_data = input_data.get(field_name)
+ current_conversion_type = self.encoding_parameters.get(field_name)
+ encoded_value = self.encode_single_field_value(field_name, current_data, current_conversion_type)
+ encoded_field_values.append(encoded_value)
+ return encoded_field_values
+
+ def encode_single_field_value(self, current_field_name, current_data, current_conversion_type):
+ if current_conversion_type is None:
+ result = current_data
+ elif current_conversion_type == INT:
+ try:
+ result = int(current_data)
+ except ValueError as e:
+ message = f"Can not convert to integer in encoding step, field name: {current_field_name}, raw data: {current_data}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif current_conversion_type == FLOAT:
+ try:
+ result = float(current_data)
+ except ValueError as e:
+ message = f"Can not convert to float in encoding step, field name: {current_field_name}, raw data: {current_data}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif current_conversion_type == LABEL_ENCODER:
+ label_encoder = self.label_encoder_by_field_name.get(current_field_name)
+ try:
+ converted_value = label_encoder.transform([current_data])
+ result = converted_value[0]
+ except ValueError as e:
+ inner_array = label_encoder.classes_
+ modified_inner_array = np.append(inner_array, [current_data])
+ label_encoder_inner_case_number=len(modified_inner_array)
+ label_encoder.classes_ = modified_inner_array
+ message = f"New item has been added to labelencoder inner classes, field name: {current_field_name}, raw data: {current_data}, inner case number: {label_encoder_inner_case_number}"
+ logger.debug(message)
+ self.label_encoder_by_field_name[current_field_name]=label_encoder
+ converted_value = label_encoder.transform([current_data])
+ result = converted_value[0]
+ except Exception as e:
+ message = f"can not convert to int using label encoder in encoding step, field name: {current_field_name}, raw data: {current_data}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ elif current_conversion_type == JULIAN:
+ try:
+ ts = pd.Timestamp(current_data)
+ result = ts.to_julian_date()
+ except Exception as e:
+ message = f"can not convert to Julian date in encoding step, field name: {current_field_name}, raw data: {current_data}"
+ logger.error(message)
+ raise InValidRequestParameterException({"message": message})
+ else:
+ message = "unknow conversion type in encoding step"
+ logger.error(message)
+ raise NotImplementedConversionTypeException({"message": message})
+ return result
+
+ def feature_engineering(self, encoded_input_data):
+ index_of_time_base_field = self.encoded_field_names.index(self.time_base_field_name)
+ time_of_input_data = encoded_input_data[index_of_time_base_field]
+ index_of_card_number = self.encoded_field_names.index(CARD_NUMBER)
+ card_number_value = encoded_input_data[index_of_card_number]
+ feature_to_be_engineered = self.feature_engineering_parameters.get(CHOSEN_FEATURE_FIELD_FOR_FEATURE_ENGINEERING)
+ index_of_feature_to_be_engineered = self.encoded_field_names.index(feature_to_be_engineered)
+ intervals = self.feature_engineering_parameters.get(INTERVALS)
+ previous_feature_and_time_base_array_by_intervals = dict()
+ for interval in intervals:
+ retrospective_time = time_of_input_data - interval
+ previous_feature_and_time_base_array = database_handler.get_feature_to_be_engineered_and_time_base_field_value_for_feature_engineering(
+ self.schema_name,
+ self.feature_engineered_table_name,
+ feature_to_be_engineered,
+ self.time_base_field_name,
+ # time_of_input_data,
+ retrospective_time,
+ card_number_value)
+ previous_feature_and_time_base_array_by_intervals[interval] = previous_feature_and_time_base_array
+ engineer = engineering_module.Engineer(self.time_base_field_name, self.encoded_field_names,
+ self.feature_engineering_parameters)
+ engineered_data_set = engineer.process(encoded_input_data, previous_feature_and_time_base_array_by_intervals,
+ index_of_feature_to_be_engineered, index_of_time_base_field)
+ database_handler.persist_feature_engineered_record(self.feature_engineered_insert_script, engineered_data_set)
+ return engineered_data_set
+
+ def predict(self, data_set):
+ try:
+ current_prediction = self.pipeline.predict([data_set])
+ current_probability = self.pipeline.predict_proba([data_set])
+ prediction = int(current_prediction[0])
+ positive_probability = float(current_probability[0][1])
+ negative_probability = float(current_probability[0][0])
+ return prediction, positive_probability, negative_probability
+ except Exception as e:
+ print(e)
+
+
+if __name__ == "__main__":
+ log_file = os.getenv("ESTIMATOR_LOG_FILE", "/home/tomi/log/estimator.log")
+ log_level = os.getenv("ESTIMATOR_LOG_LEVEL", "DEBUG")
+ # rabbitmq_url = os.getenv("RABBITMQ_URL", "localhost")
+ database_url = os.getenv("MYSQL_DATABASE_URL", "localhost")
+ database_user = os.getenv("MYSQL_USER", "root")
+ database_password = os.getenv("MYSQL_ROOT_PASSWORD", "pwd")
+ file_handler = logging.FileHandler(log_file)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ file_handler.setFormatter(formatter)
+ logger = logging.getLogger("estimator.server")
+ logger.setLevel(log_level)
+ logger.addHandler(file_handler)
+ logger.debug("MYSQL_DATABASE_URL: " + database_url)
+ logger.debug("MYSQL_USER :" + database_user)
+ logger.debug("MYSQL_PASSWORD: " + database_password)
+ database_handler = database_module.Handler(database_url, database_user, database_password)
+ estimator_holder = EstimatorHolder()
+ logger.debug("estimator modul started")
+ server.run(host='0.0.0.0', port=8083, debug=True)
--- /dev/null
+#set base image (host OS)\r
+FROM python:3.7.7\r
+\r
+# set the working directory in the container\r
+WORKDIR /code\r
+\r
+# copy the dependencies file to the working directory\r
+COPY requirements.txt .\r
+\r
+# install dependencies\r
+RUN pip install --upgrade pip && pip install -r requirements.txt\r
+\r
+# copy the content of the local src directory to the working directory\r
+COPY src/ .\r
+\r
+# command to run on container start\r
+CMD [ "python", "./server.py" ]
\ No newline at end of file
--- /dev/null
+aniso8601==9.0.1
+attrs==22.1.0
+click==8.0.3
+colorama==0.4.4
+Flask==2.0.2
+flask-restplus==0.12.1
+imbalanced-learn==0.9.0
+imblearn==0.0
+importlib-metadata==4.10.1
+importlib-resources==5.10.0
+itsdangerous==2.0.1
+Jinja2==3.0.3
+joblib==1.1.0
+jsonschema==4.16.0
+lightgbm==3.3.2
+MarkupSafe==2.1.1
+mysql-connector==2.2.9
+mysql-connector-python==8.0.28
+numpy==1.21.5
+pandas==1.3.5
+pika==1.2.0
+pkgutil_resolve_name==1.3.10
+protobuf==3.19.3
+pyrsistent==0.19.1
+python-dateutil==2.8.2
+pytz==2021.3
+scikit-learn==1.0.2
+scipy==1.7.3
+six==1.16.0
+sklearn==0.0
+threadpoolctl==3.0.0
+timestamp==0.0.1
+typing_extensions==4.0.1
+Werkzeug==2.2.2
+xgboost==1.6.2
+zipp==3.7.0
--- /dev/null
+CREATE SCHEMA IF NOT EXISTS common_fraud
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.encoded_table_registry
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ raw_dataset_id BIGINT NOT NULL,
+ encoding_id BIGINT NOT NULL,
+ time_base_field_name VARCHAR(255),
+ encoded_table_name VARCHAR(255),
+ encoded_field_names JSON,
+ label_encoder_registry JSON,
+ FOREIGN KEY (raw_dataset_id) REFERENCES raw_dataset(id),
+ FOREIGN KEY (encoding_id) REFERENCES encoding(id),
+ PRIMARY KEY (id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.encoding
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ encoding_parameters JSON,
+ primary key (id)
+) ENGINE = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.estimator
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ train_task_id BIGINT NOT NULL,
+ transaction_identifier_field_name VARCHAR(255),
+ estimator_object MEDIUMBLOB,
+ PRIMARY KEY (id),
+ FOREIGN KEY (train_task_id) REFERENCES train_task(id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.feature_engineered_table_registry
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ encoded_table_registry_id BIGINT NOT NULL,
+ feature_engineering_id BIGINT NOT NULL,
+ feature_engineered_table_name VARCHAR(255),
+ feature_engineered_insert_script VARCHAR(4096),
+ FOREIGN KEY (encoded_table_registry_id) REFERENCES encoded_table_registry(id),
+ FOREIGN KEY (feature_engineering_id) REFERENCES feature_engineering(id),
+ PRIMARY KEY (id)
+) engine = InnoDB
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.feature_engineering
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ feature_engineering_parameters JSON,
+ PRIMARY KEY (id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.fraud
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ transaction_uuid VARCHAR(255),
+ predicted BIT(1),
+ observed BIT(1),
+ predicted_value INT,
+ observed_value INT,
+ positive_probability DOUBLE PRECISION,
+ negative_probability DOUBLE PRECISION,
+ PRIMARY KEY (id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.label_encoder
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ encoder_object MEDIUMBLOB,
+ primary key (id)
+) ENGINE = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.metrics
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ estimator_id BIGINT NOT NULL,
+ TP DOUBLE PRECISION,
+ FP DOUBLE PRECISION,
+ TN DOUBLE PRECISION,
+ FN DOUBLE PRECISION,
+ sensitivity DOUBLE PRECISION,
+ specificity DOUBLE PRECISION,
+ accuracy DOUBLE PRECISION,
+ balanced_accuracy DOUBLE PRECISION,
+ prec DOUBLE PRECISION,
+ recall DOUBLE PRECISION,
+ PPV DOUBLE PRECISION,
+ NPV DOUBLE PRECISION,
+ FNR DOUBLE PRECISION,
+ FPR DOUBLE PRECISION,
+ FDR DOUBLE PRECISION,
+ F_OR DOUBLE PRECISION,
+ f1 DOUBLE PRECISION,
+ f_05 DOUBLE PRECISION,
+ f2 DOUBLE PRECISION,
+ MCC DOUBLE PRECISION,
+ ROCAUC DOUBLE PRECISION,
+ Youdens_statistic DOUBLE PRECISION,
+ PRIMARY KEY(id),
+ FOREIGN KEY (estimator_id) REFERENCES estimator(id)
+) engine = InnoDB
+
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.planned_encoding_and_feature_engineering
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ schema_name VARCHAR(255),
+ table_name VARCHAR(255),
+ detailed_information_about_table JSON,
+ time_base_field_name VARCHAR(255),
+ encoding_parameters JSON,
+ feature_engineering_parameters JSON,
+ PRIMARY KEY (id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.raw_dataset
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ schema_name VARCHAR(255),
+ table_name VARCHAR(255),
+ primary key (id)
+) ENGINE = InnoDB
\ No newline at end of file
--- /dev/null
+CREATE TABLE IF NOT EXISTS common_fraud.train_task
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ unique_field_name VARCHAR(255),
+ train_task_name VARCHAR(255),
+ planned_encoding_and_feature_engineering_id BIGINT,
+ feature_selector_code INT,
+ sampler_code INT,
+ scaler_code INT,
+ model_code INT,
+ test_size DOUBLE,
+ progress_status VARCHAR(255),
+ PRIMARY KEY (id)
+) engine = InnoDB
\ No newline at end of file
--- /dev/null
+docker run -d --hostname my-rabbit --name train_queue -p 5672:5672 -p 15672:15672 rabbitmq:3-management
\ No newline at end of file
--- /dev/null
+import json
+
+import sys
+import time
+# from datetime import datetime
+import datetime
+
+import mysql.connector
+from mysql.connector import pooling
+import numpy as np
+import logging
+import pickle
+import pandas as pd
+import csv
+import random
+import timestamp as timestamp
+
+import encoding_module
+import engineering_module
+import ddl_build_module
+
+UNDER_SCORE = "_"
+ENCODED = "encoded"
+FE = "engineered" # feature engineered
+REDUCED = "reduced"
+FOR_EVAULATED = "for_evaulated"
+SIMULATED_REAL = "simulated_real"
+DOESNT_EXIST = "doesnt_exist"
+OTHER_APPLICABLE_ENCODING_TYPES = "other_applicable_encoding_types"
+# Train task status:
+STORED = "STORED"
+IN_PROGRESS = "IN_PROGRESS"
+
+
+class IllegalArgumentException(Exception):
+ pass
+
+
+class TrainTaskExistYetException(Exception):
+ pass
+
+
+class ForeignKeyConstraintViolationException(Exception):
+ pass
+
+
+class NoneException(Exception):
+ pass
+
+
+class NumpyMySQLConverter(mysql.connector.conversion.MySQLConverter):
+ """ A mysql.connector Converter that handles Numpy types """
+
+ def _float32_to_mysql(self, value):
+ return float(value)
+
+ def _float64_to_mysql(self, value):
+ return float(value)
+
+ def _int32_to_mysql(self, value):
+ return int(value)
+
+ def _int64_to_mysql(self, value):
+ return int(value)
+
+
+class Handler:
+ def __init__(self, database_url, database_user, database_password):
+ self.logger = logging.getLogger("train.server.database")
+ self.database_url = database_url
+ self.database_user = database_user
+ self.database_password = database_password
+ # self.connection = self.get_connection(self.database_url, self.database_user, self.database_password)
+ self.connection_pool = self.get_connection_pool(self.database_url, self.database_user, self.database_password)
+
+ def get_connection_pool(self, database_url, database_user_name, database_password):
+ try:
+ connection_pool = pooling.MySQLConnectionPool(pool_name="train_pool",
+ pool_size=8,
+ pool_reset_session=True,
+ host=database_url,
+ user=database_user_name,
+ password=database_password)
+ self.logger.debug("database connection pool created in train modul")
+ return connection_pool
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL connection error: {err.msg}")
+
+ # def get_connection(self, database_url, database_user_name, database_password):
+ # try:
+ # connection = mysql.connector.connect(
+ # # pool_name="local",
+ # # pool_size=16,
+ # host=database_url,
+ # user=database_user_name,
+ # password=database_password)
+ # self.logger.debug("database connection created in train modul")
+ # connection.set_converter_class(NumpyMySQLConverter)
+ # return connection
+ # except mysql.connector.Error as err:
+ # self.logger.error(f"MySQL connection error: {err.msg}")
+
+ def create_common_fraud_schemas(self):
+
+ commands = ["SQL CREATE SCHEMA common_fraud.txt",
+ "SQL CREATE TABLE planned_encoding_and_feature_engineering.txt",
+ "SQL CREATE TABLE raw_dataset.txt",
+ "SQL CREATE TABLE encoding.txt",
+ "SQL CREATE TABLE feature_engineering.txt",
+ "SQL CREATE TABLE encoded_table_registry.txt",
+ "SQL CREATE TABLE feature_engineered_table_registry.txt",
+ "SQL CREATE TABLE train_task.txt",
+ "SQL CREATE TABLE label_encoder.txt",
+ "SQL CREATE TABLE estimator.txt",
+ "SQL CREATE TABLE metrics.txt"
+ ]
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ for command in commands:
+ try:
+ file = open(command, "r")
+ sql_create_script = file.read()
+ cursor.execute(sql_create_script)
+ connection_object.commit()
+ except FileNotFoundError:
+ self.logger.error(f"SQL script file not found {command}")
+ except OSError:
+ self.logger.error("OS Error")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}, file: {file}")
+ cursor.close()
+ connection_object.close()
+
+ def upload_train_database(self, request_parameter):
+ schema_name = request_parameter.get("schema_name")
+ if schema_name is None:
+ raise IllegalArgumentException({"message": "Parameter is missing in request body",
+ "parameter_name": "schema_name"})
+ table_name = request_parameter.get("table_name")
+ if table_name is None:
+ raise IllegalArgumentException({"message": "Parameter is missing in request body",
+ "parameter_name": "table_name"})
+ field_type_by_name = request_parameter.get("fields")
+ if field_type_by_name is None:
+ raise IllegalArgumentException({"message": "Parameter is missing in request body",
+ "parameter_name": "fields"})
+ if type(field_type_by_name) is not dict:
+ raise IllegalArgumentException({"message": "The parameter isn't dictionary",
+ "parameter_name": "fields"})
+ path = request_parameter.get("path")
+ if path is None:
+ raise IllegalArgumentException({"message": "Parameter is missing in request body",
+ "parameter_name": "path"})
+ csv_file_name = request_parameter.get("csv_file_name")
+ if csv_file_name is None:
+ raise IllegalArgumentException({"message": "Parameter is missing in request body",
+ "parameter_name": "csv_file_name"})
+ csv_filename_with_path = path + "/" + csv_file_name
+ try:
+ data = pd.read_csv(csv_filename_with_path, delimiter=';', na_filter=None)
+ df = pd.DataFrame(data)
+ train_as_array = df.to_numpy()
+ train_to_persist = train_as_array[1:, ]
+ except FileNotFoundError:
+ raise IllegalArgumentException({"message": "CSV file not found"})
+ except OSError:
+ raise IllegalArgumentException({"message": "OS error occured"})
+ self.create_train_schema(schema_name)
+ self.create_train_table(schema_name, table_name, field_type_by_name, True)
+ ddl_command_builder = ddl_build_module.DdlCommandBuilder()
+ insert_script = ddl_command_builder.create_insert_into_generic_train_database_script(schema_name, table_name,
+ field_type_by_name)
+ self.persist_encoded_or_engineered_dataset(insert_script, train_to_persist, "train")
+
+ def create_train_schema(self, schema_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = f"CREATE SCHEMA IF NOT EXISTS {schema_name}"
+ try:
+ cursor.execute(sql_insert_command)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"Train schema created, schema name: {schema_name}")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def create_train_table(self, schema_name, table_name, field_type_by_name, is_id_auto_incremented):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ ddl_command_builder = ddl_build_module.DdlCommandBuilder()
+ sql_insert_command = ddl_command_builder.build_create_generic_train_database_script(schema_name,
+ table_name,
+ field_type_by_name,
+ is_id_auto_incremented)
+ try:
+ cursor.execute(sql_insert_command)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"Train table created, schema name: {schema_name}, table name: {table_name}")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_schema_names(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SHOW SCHEMAS"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ schema_names = list()
+ for item in query_result:
+ schema_names.append(item[0])
+ self.logger.debug(f"schema names: {schema_names}")
+ return schema_names
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_table_names_of_given_database(self, schema_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT distinct TABLE_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}'"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ table_names = list()
+ for item in query_result:
+ table_names.append(item[0])
+ self.logger.debug(f"tables names of {schema_name} schema: {table_names}")
+ return table_names
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_records(self, database_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ result = dict()
+ sql_select_query = f"SELECT * FROM {database_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ for record in query_result:
+ fields = list()
+ length = len(record)
+ id = 0
+ for index in range(length):
+ if index == 0:
+ id = record[index]
+ else:
+ fields.append(record[index])
+ result[id] = fields
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_records_with_limit_and_offset(self, database_name, table_name, limit, offset):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT * FROM {database_name}.{table_name}" + (
+ f" limit {limit}" if limit is not None else "") + (f" offset {offset}" if offset is not None else "")
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ records = list()
+ for item in query_result:
+ fields = list()
+ length = len(item)
+ for index in range(length):
+ field = item[index]
+ type_of_field = type(field)
+ if type_of_field == datetime:
+ field = datetime.strftime(field, '%Y-%m-%d %H:%M:%S:%f')
+ fields.append(field)
+ records.append(fields)
+ return records
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_detailed_information_about_table(self, schema_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ result = dict()
+ column_properties = list()
+ if table_name is None:
+ self.logger.debug("No table name set, query the transaction table")
+ table_name = "transaction"
+ sql_select_query = f"SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}' ORDER BY ordinal_position"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ if query_result:
+ for column_name_and_type in query_result:
+ column_property = dict()
+ column_name = column_name_and_type[0]
+ column_type_as_byte = column_name_and_type[1]
+ column_type = column_type_as_byte.decode("utf-8")
+ column_property["name"] = column_name
+ column_property["type"] = column_type
+ column_properties.append(column_property)
+ result["fields"] = column_properties
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ sql_select_query = f"SELECT COUNT(*) FROM {schema_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchone()
+ result["record_number"] = query_result[0] if query_result else 0
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ sql_select_query = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}' AND COLUMN_KEY='PRI'"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchone()
+ result["primary_key"] = query_result[0] if query_result else ""
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ cursor.close()
+ connection_object.close()
+ fraud_candidates = self.get_fraud_candidate_column_and_fraud_number(schema_name, table_name,
+ result)
+ result["fraud_candidates"] = fraud_candidates
+ self.get_other_proper_column_data_type_to_encoding(schema_name, table_name, result)
+ return result
+
+ def get_fraud_candidate_column_and_fraud_number(self, schema_name, table_name, database_property_holder):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ candidate_fraud_columns = list()
+ column_names_and_types = database_property_holder.get("fields")
+ for column_name_and_type in column_names_and_types:
+ column_name = column_name_and_type.get("name")
+ sql_select_query = f"SELECT {column_name} FROM {schema_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ candidate_fraud_column = dict()
+ record_number = len(query_result)
+ number_of_ones = 0
+ number_of_nulls = 0
+ for field in query_result:
+ if field[0] == 1 or field[0] == '1':
+ number_of_ones = number_of_ones + 1
+ elif field[0] == 0 or field[0] == '0':
+ number_of_nulls = number_of_nulls + 1
+ if number_of_ones + number_of_nulls == record_number and number_of_ones != 0 and number_of_nulls != 0:
+ candidate_fraud_column["name"] = column_name
+ candidate_fraud_column["fraud_number"] = number_of_ones
+ candidate_fraud_column["no_fraud_number"] = number_of_nulls
+ candidate_fraud_columns.append(candidate_fraud_column)
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ cursor.close()
+ connection_object.close()
+ return candidate_fraud_columns
+
+ def get_other_proper_column_data_type_to_encoding(self, schema_name, table_name, database_property_holder):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ column_names_and_types = database_property_holder.get("fields")
+ for column_name_and_type in column_names_and_types:
+ column_name = column_name_and_type.get("name")
+ column_type = column_name_and_type.get("type")
+ other_applicable_encoding_types = list()
+ if column_type.startswith("varchar"):
+ other_applicable_encoding_types.clear()
+ sql_select_query = f"SELECT {column_name} FROM {schema_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ records = cursor.fetchall()
+ is_records_number_type = True
+ is_records_suited_int_type = True
+ for record_as_tuple in records:
+ record = record_as_tuple[0]
+ if not record.isnumeric():
+ is_records_number_type = False
+ is_records_suited_int_type = False
+ break
+ elif self.is_int(record):
+ value = int(record)
+ if value > 2147483647:
+ is_records_suited_int_type = False
+ else:
+ continue
+ elif self.is_float(record):
+ is_records_suited_int_type = False
+
+ if is_records_number_type:
+ other_applicable_encoding_types.append("float")
+ if is_records_suited_int_type:
+ other_applicable_encoding_types.append("int")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ column_name_and_type[OTHER_APPLICABLE_ENCODING_TYPES] = other_applicable_encoding_types
+ cursor.close()
+ connection_object.close()
+
+ def is_int(self, element):
+ try:
+ int(element)
+ return True
+ except ValueError:
+ return False
+
+ def is_float(self, element):
+ try:
+ float(element)
+ return True
+ except ValueError:
+ return False
+
+ def persist_encoding_and_engineering_plan(self, schema_name, table_name, detailed_information_about_table,
+ used_time_base_field_name,
+ encoding_parameters,
+ feature_engineering_parameters):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_query = "INSERT INTO common_fraud.planned_encoding_and_feature_engineering (schema_name, table_name, detailed_information_about_table, time_base_field_name ,encoding_parameters, feature_engineering_parameters) VALUES (%s,%s,%s,%s,%s,%s)"
+ parameter = (
+ schema_name, table_name, json.dumps(detailed_information_about_table), used_time_base_field_name,
+ json.dumps(encoding_parameters),
+ json.dumps(feature_engineering_parameters))
+ try:
+ cursor.execute(sql_insert_query, parameter)
+ last_inserted_row_id = cursor.lastrowid
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(
+ f"encoding and feature enginnering plan persisted, schema name: {schema_name}, table name: {table_name}, encoding parameters: {encoding_parameters}, feature engineering parameters {feature_engineering_parameters}")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ return last_inserted_row_id
+
+ def get_all_encoding_and_feature_engineering_plan(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ response = dict()
+ sql_select_query = "SELECT * FROM common_fraud.planned_encoding_and_feature_engineering"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ response = None
+ if query_result is not None:
+ response = self.build_encoding_and_feature_engineering_plan_response(query_result)
+ return response
+
+ # 2.1
+ def get_encoding_and_feature_engineering_plan_by_id(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.planned_encoding_and_feature_engineering WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ if query_result is None:
+ raise NoneException({"message": "The given id doesn't exist", "parameter": id})
+ response = self.build_encoding_and_feature_engineering_plan_response(query_result)
+ return response
+
+ def delete_encoding_and_feature_engineering_plan(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_delete_command = "DELETE FROM common_fraud.planned_encoding_and_feature_engineering WHERE id= %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_delete_command, parameter)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def build_encoding_and_feature_engineering_plan_response(self, query_result):
+ response = list()
+ for record in query_result:
+ single_response = dict()
+ single_response["id"] = record[0]
+ single_response["schema_name"] = record[1]
+ single_response["table_name"] = record[2]
+ single_response["detailed_information_about_table"] = json.loads(record[3])
+ single_response["time_base_field_name"] = record[4]
+ single_response["encoding_parameters"] = json.loads(record[5])
+ single_response["feature_engineering_parameters"] = json.loads(record[6])
+ response.append(single_response)
+ return response
+
+ def persist_train_task(self, request_parameter, feature_selector_keys, samplers_keys, scalers_keys, models_keys):
+ unique_field_name = request_parameter.get("unique_field_name")
+ if unique_field_name is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body",
+ "parameter_name": "transaction_identifier_field_name"})
+ name = request_parameter.get("train_task_name")
+ if name is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body",
+ "parameter_name": {name}})
+ planned_encoding_and_feature_engineering_id = request_parameter.get(
+ "planned_encoding_and_feature_engineering_id")
+ if planned_encoding_and_feature_engineering_id is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body",
+ "parameter_name": "planned_encoding_and_feature_engineering_id"})
+ existing_planned_encoding_id_and_feature_engineering_id = self.get_existing_planned_encoding_and_feature_engineering_id()
+ if planned_encoding_and_feature_engineering_id not in existing_planned_encoding_id_and_feature_engineering_id:
+ raise IllegalArgumentException(
+ {"message": "Parameter is invalid", "parameter_name": "planned_encoding_and_feature_engineering_id",
+ "value": planned_encoding_and_feature_engineering_id})
+
+ feature_selector_code = request_parameter.get("feature_selector_code")
+ if feature_selector_code is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body", "parameter_name": "feature selector code"})
+ if feature_selector_code not in feature_selector_keys:
+ raise IllegalArgumentException(
+ {"message": "Parameter is invalid", "parameter_name": "feature_selector_code",
+ "value": feature_selector_code})
+
+ sampler_code = request_parameter.get("sampler_code")
+ if sampler_code is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body", "parameter_name": "sampler_code"})
+ if sampler_code not in samplers_keys:
+ raise IllegalArgumentException(
+ {"message": "Parameter is invalid", "parameter_name": "sampler_code",
+ "value": sampler_code})
+
+ scaler_code = request_parameter.get("scaler_code")
+ if scaler_code is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body", "parameter_name": "scaler_code"})
+ if scaler_code not in scalers_keys:
+ raise IllegalArgumentException(
+ {"message": "Parameter is invalid", "parameter_name": "scaler_code",
+ "value": scaler_code})
+
+ model_code = request_parameter.get("model_code")
+ if model_code is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body", "parameter_name": "model_code"})
+ if model_code not in models_keys:
+ raise IllegalArgumentException(
+ {"message": "Parameter is invalid", "parameter_name": "model_code",
+ "value": model_code})
+
+ test_size = request_parameter.get("test_size")
+ if test_size is None:
+ raise IllegalArgumentException(
+ {"message": "Parameter is missing in request body", "parameter_name": "test_size"})
+ if test_size >= 1 or test_size <= 0:
+ raise IllegalArgumentException(
+ {"message": "Parameter must be between 0 and 1", "parameter_name": "test_size", "value": test_size})
+
+ progress_status = STORED
+
+ self.check_if_train_task_exist(name, planned_encoding_and_feature_engineering_id, feature_selector_code,
+ sampler_code, scaler_code,
+ model_code, test_size)
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_query = "INSERT INTO common_fraud.train_task (unique_field_name, train_task_name, planned_encoding_and_feature_engineering_id, feature_selector_code, sampler_code, scaler_code, model_code, test_size, progress_status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
+ parameter = (
+ unique_field_name, name, planned_encoding_and_feature_engineering_id, feature_selector_code,
+ sampler_code, scaler_code, model_code, test_size, progress_status)
+ try:
+ cursor.execute(sql_insert_query, parameter)
+ connection_object.commit()
+ last_inserted_row_id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(
+ f"Train task persisted, train_task_name: {name},planned_encoding_id_and_feature_engineering_id: {planned_encoding_and_feature_engineering_id}, feature_selector_code: {feature_selector_code}, sampler_code: {sampler_code}, scaler_code: {scaler_code}, model_code: {model_code}, test_size: {test_size}")
+ return last_inserted_row_id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def check_if_train_task_exist(self, name, planned_encoding_and_feature_engineering_id, feature_selector_code,
+ sampler_code, scaler_code, model_code, test_size):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.train_task WHERE train_task_name = %s and planned_encoding_and_feature_engineering_id = %s and feature_selector_code = %s and sampler_code = %s and scaler_code = %s and model_code = %s and test_size = %s"
+ parameter = (
+ name, planned_encoding_and_feature_engineering_id, feature_selector_code, sampler_code, scaler_code,
+ model_code, test_size)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ if query_result is not None:
+ id = query_result[0]
+ raise TrainTaskExistYetException({"message": "Train task exists yet", "id": id})
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_all_train_task(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.train_task"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = self.build_train_task_response(query_result)
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_train_task_by_id(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.train_task WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = self.build_train_task_response(query_result)
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def build_train_task_response(self, query_result):
+ result = dict()
+ for item in query_result:
+ id = item[0]
+ unique_field_name = item[1]
+ name = item[2]
+ planned_encoding_and_feature_engineering_id = item[3]
+ feature_selector_code = item[4]
+ sampler_code = item[5]
+ scaler_code = item[6]
+ model_code = item[7]
+ test_size = item[8]
+ process_status = item[9]
+ result[id] = {
+ "unique_field_name": unique_field_name,
+ "train_task_name": name,
+ "planned_encoding_and_feature_engineering_id": planned_encoding_and_feature_engineering_id,
+ "feature_selector_code": feature_selector_code,
+ "sampler_code": sampler_code,
+ "scaler_code": scaler_code,
+ "model_code": model_code,
+ "test_size": test_size,
+ "process_status": process_status}
+ return result
+
+ def set_train_task_status(self, id, progress_status):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_update_command = "UPDATE common_fraud.train_task SET progress_status = %s WHERE id = %s"
+ parameter = (progress_status, id)
+ try:
+ cursor.execute(sql_update_command, parameter)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_train_task_status(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT progress_status FROM common_fraud.train_task WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ if query_result is not None:
+ status = query_result[0]
+ else:
+ status = DOESNT_EXIST
+ return status
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def create_train_task_csv_file(self, path, file_name, date_time_to_append, feature_selectors, samplers, scalers,
+ models):
+ header_elements, lines = self.create_train_task_csv_content(feature_selectors, samplers, scalers, models)
+ self.write_csv_file(path, file_name, date_time_to_append, header_elements, lines)
+
+ def create_train_task_csv_content(self, feature_selectors, samplers, scalers, models):
+ header_elements = (
+ "id", "transaction_identifier_field_name", "train_task_name", "planned_encoding_and_feature_engineering",
+ "feature_selector", "sampler", "scaler", "model", "test_size", "process_status")
+ lines = list()
+ train_task_properties_by_id = self.get_all_train_task()
+ for key, train_task_properties in train_task_properties_by_id.items():
+ line = list()
+ id = key
+ transaction_identifier_field_name = train_task_properties.get("transaction_identifier_field_name")
+ train_task_name = train_task_properties.get("train_task_name")
+ planned_encoding_and_feature_engineering_id = train_task_properties.get(
+ "planned_encoding_and_feature_engineering_id")
+ feature_selector_code = train_task_properties.get("feature_selector_code")
+ sampler_code = train_task_properties.get("sampler_code")
+ scaler_code = train_task_properties.get("scaler_code")
+ model_code = train_task_properties.get("model_code")
+ test_size = train_task_properties.get("test_size")
+ progress_status = train_task_properties.get("process_status")
+ planned_encoding_and_feature_engineering_dict = self.get_encoding_and_feature_engineering_plan_by_id(
+ planned_encoding_and_feature_engineering_id)
+ planned_encoding_and_feature_engineering_string = self.convert_encoding_and_feature_enginnering_dict_to_string(
+ planned_encoding_and_feature_engineering_dict[0])
+ line.append(id)
+ line.append(transaction_identifier_field_name)
+ line.append(train_task_name)
+ line.append(planned_encoding_and_feature_engineering_string)
+ line.append(feature_selectors.get(feature_selector_code)[0])
+ line.append(samplers.get(sampler_code)[0])
+ line.append(scalers.get(scaler_code)[0])
+ line.append(models.get(model_code)[0])
+ line.append(test_size)
+ line.append(progress_status)
+ lines.append(line)
+ return header_elements, lines
+
+ def create_metrics_csv_file(self, path, file_name, date_time_to_append):
+ header_elements, lines = self.create_metrics_csv_content()
+ self.write_csv_file(path, file_name, date_time_to_append, header_elements, lines)
+
+ def create_metrics_csv_content(self):
+ header_elements = (
+ "id", "train_task_name", "TP", "FP", "TN", "FN", "accuracy", "balanced_accuracy", "precision", "recall",
+ "sensitivity", "specificity", "PPV", "NPV", "FNR", "FPR", "FDR", "FOR", "f1", "f0.5", "f2", "MCC", "ROCAUC",
+ "Youdens_statistic")
+ lines = list()
+ list_of_metrics_dict = self.get_all_metrics()
+ for single_metrics_dict in list_of_metrics_dict:
+ line = list()
+ id = single_metrics_dict.get("id")
+ estimator_id = single_metrics_dict.get("estimator_id")
+ TP = single_metrics_dict.get("TP")
+ FP = single_metrics_dict.get("FP")
+ TN = single_metrics_dict.get("TN")
+ FN = single_metrics_dict.get("FN")
+ accuracy = single_metrics_dict.get("accuracy")
+ balanced_accuracy = single_metrics_dict.get("balanced_accuracy")
+ precision = single_metrics_dict.get("precision")
+ recall = single_metrics_dict.get("recall")
+ sensitivity = single_metrics_dict.get("sensitivity")
+ specificity = single_metrics_dict.get("specificity")
+ PPV = single_metrics_dict.get("PPV")
+ NPV = single_metrics_dict.get("NPV")
+ FNR = single_metrics_dict.get("FNR")
+ FPR = single_metrics_dict.get("FPR")
+ FDR = single_metrics_dict.get("FDR")
+ FOR = single_metrics_dict.get("FOR")
+ f1 = single_metrics_dict.get("f1")
+ f0_5 = single_metrics_dict.get("f0.5")
+ f2 = single_metrics_dict.get("f2")
+ MCC = single_metrics_dict.get("MCC")
+ ROCAUC = single_metrics_dict.get("ROCAUC")
+ Youdens_statistic = single_metrics_dict.get("Youdens_statistic")
+ train_task_name = self.get_train_task_name_by_estimator_id(estimator_id)
+ line.append(id)
+ line.append(train_task_name)
+ line.append(TP)
+ line.append(FP)
+ line.append(TN)
+ line.append(FN)
+ line.append(accuracy)
+ line.append(balanced_accuracy)
+ line.append(precision)
+ line.append(recall)
+ line.append(sensitivity)
+ line.append(specificity)
+ line.append(PPV)
+ line.append(NPV)
+ line.append(FNR)
+ line.append(FPR)
+ line.append(FDR)
+ line.append(FOR)
+ line.append(f1)
+ line.append(f0_5)
+ line.append(f2)
+ line.append(MCC)
+ line.append(ROCAUC)
+ line.append(Youdens_statistic)
+ lines.append(line)
+ return header_elements, lines
+
+ def convert_encoding_and_feature_enginnering_dict_to_string(self, input_dict):
+ keys = (
+ "schema_name", "table_name", "time_base_field_name", "encoding_parameters",
+ "feature_engineering_parameters")
+ line = str()
+ for key in keys:
+ line += key + " = "
+ value = input_dict.get(key)
+ if isinstance(value, list) or isinstance(value, dict):
+ flatted_value = str(value)
+ converted_flatted_value = flatted_value.replace(",", ";")
+ line += converted_flatted_value
+ else:
+ line += value if value is not None else "null"
+ if keys.index(key) != len(keys) - 1:
+ line += "; "
+ return line
+
+ def write_csv_file(self, path, file_name, date_time_to_append, header_elements, lines):
+ full_file_name = path + "/" + file_name + ".csv"
+ if date_time_to_append == True:
+ datetime_now = datetime.datetime.now()
+ year_month_day = datetime_now.strftime("%Y%m%d")
+ hour_min_sec = datetime_now.strftime("%H%M%S")
+ full_file_name = full_file_name + "_" + str(year_month_day) + "_" + str(hour_min_sec)
+ with open(full_file_name, 'w') as csv_file:
+ csv_writer = csv.writer(csv_file)
+ csv_writer.writerow(header_elements)
+ csv_writer.writerows(lines)
+
+ def get_train_task_name_by_estimator_id(self, estimator_id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT train_task_name FROM common_fraud.train_task t join common_fraud.estimator e on t.id = e.train_task_id WHERE e.id = %s"
+ parameter = (estimator_id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ return query_result[0] if query_result else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def check_if_any_train_in_progress(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.train_task WHERE progress_status = %s"
+ parameter = (IN_PROGRESS,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ if query_result:
+ result = True
+ else:
+ result = False
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_existing_planned_encoding_and_feature_engineering_id(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ result = list()
+ sql_select_query = "select id from common_fraud.planned_encoding_and_feature_engineering"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ for item in query_result:
+ result.append(item[0])
+ return result
+
+ def get_parameters_from_plan_by_id(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.planned_encoding_and_feature_engineering WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ schema_name = query_result[1]
+ table_name = query_result[2]
+ detailed_information_about_table = json.loads(query_result[3])
+ time_base_field_name = query_result[4]
+ encoding_parameters = json.loads(query_result[5])
+ feature_engineering_parameters = json.loads(query_result[6])
+ return schema_name, table_name, detailed_information_about_table, time_base_field_name, encoding_parameters, feature_engineering_parameters
+
+ def get_raw_dataset_id(self, schema_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.raw_dataset WHERE schema_name = %s and table_name = %s"
+ parameter = (schema_name, table_name)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ if query_result is not None:
+ id = query_result[0]
+ else:
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.raw_dataset (schema_name, table_name) VALUES(%s, %s)"
+ parameter = (schema_name, table_name)
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ return id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_encoding_id_by_json_content(self, encoding_parameters):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.encoding"
+ try:
+ id = None
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ for item in query_result:
+ if json.loads(item[1]) == encoding_parameters:
+ id = item[0]
+ if id is None:
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.encoding (encoding_parameters) VALUES(%s)"
+ parameter = (json.dumps(encoding_parameters),)
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ return id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_encoded_table_properties(self, raw_dataset_id, encoding_id, time_base_field_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.encoded_table_registry where raw_dataset_id = %s and encoding_id = %s and time_base_field_name = %s"
+ try:
+ parameter = (raw_dataset_id, encoding_id, time_base_field_name)
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ id = None
+ encoded_table_name = None
+ encoded_field_names = None
+ label_encoder_registry = None
+ if query_result is not None:
+ id = query_result[0]
+ encoded_table_name = query_result[4]
+ encoded_field_names = json.loads(query_result[5])
+ label_encoder_registry = json.loads(query_result[6])
+ return id, encoded_table_name, encoded_field_names, label_encoder_registry
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_feature_engineering_id_by_json_content(self, feature_engineering_parameters):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.feature_engineering"
+ try:
+ id = None
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ for item in query_result:
+ if json.loads(item[1]) == feature_engineering_parameters:
+ id = item[0]
+ if id is None:
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.feature_engineering (feature_engineering_parameters) VALUES(%s)"
+ parameter = (json.dumps(feature_engineering_parameters),)
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ return id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def encode(self, raw_dataset_id, schema_name, table_name, encoding_id, encoding_parameters, time_base_field_name):
+
+ detailed_information_about_table = self.get_detailed_information_about_table(schema_name, table_name)
+ field_types_by_name = self.get_field_types_by_name(schema_name, table_name)
+ original_field_names = list(field_types_by_name.keys())
+ original_data_set = self.get_record_for_processing(schema_name, table_name, time_base_field_name)
+ encoder = encoding_module.DataBaseEncoder(original_data_set, original_field_names)
+ encoded_array, encoder_by_field_name, encoded_field_names = encoder.encode(encoding_parameters,
+ detailed_information_about_table)
+ encoded_table_name = table_name + UNDER_SCORE + ENCODED + UNDER_SCORE + str(encoding_id)
+ ddl_command_builder = ddl_build_module.DdlCommandBuilder()
+ encoded_table_create_script = ddl_command_builder.build_create_encoded_or_engineered_table_script(schema_name,
+ encoded_table_name,
+ encoded_field_names)
+ self.create_encoded_or_engineered_table(encoded_table_create_script)
+ insert_script = ddl_command_builder.create_insert_into_encoded_or_feature_engineered_script(schema_name,
+ encoded_table_name,
+ encoded_field_names)
+ self.persist_encoded_or_engineered_dataset(insert_script, encoded_array, type="Encoded")
+ label_encoder_registry = dict()
+ for field_name, label_encoder in encoder_by_field_name.items():
+ id_in_label_encoder_table = self.persist_label_encoder(field_name, label_encoder)
+ label_encoder_registry[field_name] = id_in_label_encoder_table
+ encoded_table_registry_id = self.persist_encoded_table_registry(raw_dataset_id, encoding_id,
+ time_base_field_name, encoded_table_name,
+ encoded_field_names, label_encoder_registry)
+ return encoded_array, encoded_table_registry_id, encoded_table_name, encoded_field_names, label_encoder_registry
+
+ def get_field_types_by_name(self, schema_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}' ORDER BY ordinal_position"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ field_types_by_name = dict()
+ for column_name_and_type in query_result:
+ field_name = column_name_and_type[0]
+ field_type = column_name_and_type[1].decode("utf-8")
+ field_types_by_name[field_name] = field_type
+ self.logger.debug(f"field types by name in {schema_name} schema, {table_name} table: {field_types_by_name}")
+ return field_types_by_name
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_field_name_in_ordinal_position(self, schema_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}' ORDER BY ordinal_position"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ field_names = list()
+ for item in query_result:
+ field_names.append(item[0])
+ self.logger.debug(f"field names in {schema_name} schema, {table_name} table: {field_names}")
+ return field_names
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_field_type_in_ordinal_position(self, schema_name, table_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}' ORDER BY ordinal_position"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ field_names = list()
+ for item in query_result:
+ field_names.append(item[0].decode("utf-8"))
+ self.logger.debug(f"field types in {schema_name} schema, {table_name} table: {field_names}")
+ return field_names
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_record_for_processing(self, schema_name, table_name, used_time_base_field):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT * FROM {schema_name}.{table_name} ORDER BY {used_time_base_field} DESC"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = np.array(query_result)
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_encoded_or_engineered_dataset(self, script, dataset, type):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ length_of_dataset = len(dataset)
+ bound = 100
+ try:
+ if length_of_dataset > bound:
+ number_of_part_array = int(length_of_dataset / bound)
+ number_of_rest_datas = length_of_dataset - number_of_part_array * bound
+ for i in range(0, number_of_part_array, 1):
+ temp_array = dataset[i * bound:(i + 1) * bound, :]
+ value_list = list()
+ for record in temp_array:
+ value_list.append(tuple(record))
+ cursor.executemany(script, value_list)
+ connection_object.commit()
+ temp_array = dataset[
+ (number_of_part_array) * bound:(number_of_part_array) * bound + number_of_rest_datas, :]
+ value_list = list()
+ for record in temp_array:
+ value_list.append(tuple(record))
+ cursor.executemany(script, value_list)
+ connection_object.commit()
+ else:
+ value_list = list()
+ for record in dataset:
+ value_list.append(tuple(record))
+ cursor.executemany(script, value_list)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"{type} database persisted")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_table_name_from_feature_engineering_registry(self, encoded_table_registry_id, feature_engineering_id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.feature_engineered_table_registry WHERE encoded_table_registry_id = %s and feature_engineering_id = %s"
+ try:
+ parameter = (encoded_table_registry_id, feature_engineering_id)
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ feature_engineered_table_name = None
+ feature_engineered_insert_script = None
+ if query_result is not None:
+ feature_engineered_table_name = query_result[3]
+ feature_engineered_insert_script = query_result[4]
+ return feature_engineered_table_name, feature_engineered_insert_script
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def create_encoded_or_engineered_table(self, script):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ try:
+ cursor.execute(script)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_records_for_processing(self, schema_name, table_name, time_stamp_field_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT * FROM {schema_name}.{table_name} ORDER BY {time_stamp_field_name} DESC"
+ try:
+ cursor.execute(sql_select_query)
+ result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ records_with_id = np.array(result)
+ records_without_id = records_with_id[:, 1:]
+ return records_without_id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_label_encoder(self, field_name, encoder):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.label_encoder (encoder_object) values (%s)"
+ pickled_encoder = pickle.dumps(encoder)
+ parameter = (pickled_encoder,)
+ try:
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ last_inserted_row_id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"label encoder persisted, field name: {field_name}, id: {id}")
+ return last_inserted_row_id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_encoded_table_registry(self, raw_dataset_id, encoding_id, time_base_field_name, encoded_table_name,
+ encoded_field_names,
+ label_encoder_registry):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.encoded_table_registry (raw_dataset_id, encoding_id, time_base_field_name, encoded_table_name, encoded_field_names, label_encoder_registry) values (%s,%s,%s,%s,%s,%s)"
+ parameter = (
+ raw_dataset_id, encoding_id, time_base_field_name, encoded_table_name, json.dumps(encoded_field_names),
+ json.dumps(label_encoder_registry))
+ try:
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ last_inserted_row_id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(
+ f"record persisted in the encoded_table_registry table, encoded_table_name: {encoded_table_name}")
+ return last_inserted_row_id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def feature_engineer(self, schema_name, encoded_table_name, fraud_type_field_name, feature_engineering_id, dataset,
+ time_base_field_name,
+ encoded_field_names, feature_engineering_parameters, encoded_table_registry_id, used_cpu_core):
+ engineer = engineering_module.Engineer(time_base_field_name, encoded_field_names, fraud_type_field_name,
+ feature_engineering_parameters,
+ used_cpu_core)
+ feature_engineered_dataset, feature_engineered_field_names = engineer.create_new_features(dataset)
+ feature_engineered_table_name = encoded_table_name + UNDER_SCORE + FE + UNDER_SCORE + str(
+ feature_engineering_id)
+ ddl_command_builder = ddl_build_module.DdlCommandBuilder()
+ feature_engineered_table_create_script = ddl_command_builder.build_create_encoded_or_engineered_table_script(
+ schema_name, feature_engineered_table_name, feature_engineered_field_names)
+ self.create_encoded_or_engineered_table(feature_engineered_table_create_script)
+ feature_engineered_insert_script = ddl_command_builder.create_insert_into_encoded_or_feature_engineered_script(
+ schema_name, feature_engineered_table_name, feature_engineered_field_names)
+ feature_engineered_insert_script_using_in_prediction = ddl_command_builder.create_insert_into_encoded_or_feature_engineered_script_for_prediction(
+ schema_name, feature_engineered_table_name, feature_engineered_field_names, fraud_type_field_name)
+ self.persist_encoded_or_engineered_dataset(feature_engineered_insert_script, feature_engineered_dataset,
+ type="Feature engineered")
+ self.persist_feature_engineered_table_registry_item(encoded_table_registry_id, feature_engineering_id,
+ feature_engineered_table_name,
+ feature_engineered_insert_script)
+ return feature_engineered_dataset, feature_engineered_table_name, feature_engineered_insert_script_using_in_prediction
+
+ def persist_feature_engineered_table_registry_item(self, encoded_table_registry_id, feature_engineering_id,
+ feature_engineered_table_name, feature_engineered_insert_script):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_command = "INSERT INTO common_fraud.feature_engineered_table_registry (encoded_table_registry_id, feature_engineering_id, feature_engineered_table_name, feature_engineered_insert_script) values (%s,%s,%s,%s)"
+ parameter = (encoded_table_registry_id, feature_engineering_id, feature_engineered_table_name,
+ feature_engineered_insert_script)
+ try:
+ cursor.execute(sql_insert_command, parameter)
+ connection_object.commit()
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(
+ f"record persisted in feature_engineered_table_registry table, feature_engineered_table_name: {feature_engineered_table_name}")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ #
+ def get_train_parameters(self, train_task_id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "select * from common_fraud.train_task where id = %s"
+ parameter = (train_task_id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_estimator(self, train_task_id, transaction_identifier_field_name, estimator):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_query = "INSERT INTO common_fraud.estimator (train_task_id, transaction_identifier_field_name, estimator_object) VALUES (%s,%s,%s)"
+ pickled_estimator_container = pickle.dumps(estimator)
+ parameter = (train_task_id, transaction_identifier_field_name, pickled_estimator_container)
+ try:
+ cursor.execute(sql_insert_query, parameter)
+ connection_object.commit()
+ last_inserted_row_id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"estimator persisted, train_task_id: {train_task_id}, id: {last_inserted_row_id}")
+ return last_inserted_row_id
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def persist_metrics(self, estimator_id, TP, FP, TN, FN, sensitivity, specificity, accuracy, balanced_accuracy,
+ precision, recall, PPV, NPV, FNR, FPR, FDR, FOR, f1, f_05, f2, MCC, ROCAUC, Youdens_statistic):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_insert_query = "INSERt INTO common_fraud.metrics (estimator_id,TP,FP,TN,FN,sensitivity,specificity,accuracy,balanced_accuracy,prec,recall,PPV,NPV,FNR,FPR,FDR,F_OR,f1,f_05,f2,MCC,ROCAUC,Youdens_statistic) VALUES" \
+ "(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
+ values = (estimator_id, TP, FP, TN, FN, sensitivity, specificity, accuracy, balanced_accuracy,
+ precision, recall, PPV, NPV, FNR, FPR, FDR, FOR, f1, f_05, f2, MCC, ROCAUC, Youdens_statistic)
+ try:
+ cursor.execute(sql_insert_query, values)
+ connection_object.commit()
+ last_inserted_row_id = cursor.lastrowid
+ cursor.close()
+ connection_object.close()
+ self.logger.debug(f"metrics persisted, estimator_id: {estimator_id}, id: {last_inserted_row_id}")
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_all_metrics(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.metrics"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ single_response = self.build_complex_metrics_response(query_result)
+ return single_response
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ # 6
+ def get_metrics_by_id(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.metrics WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ complex_response = self.build_complex_metrics_response(query_result)
+ return complex_response
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def build_complex_metrics_response(self, query_result):
+ response = list()
+ for record in query_result:
+ response.append(self.build_simple_metrics_response(record))
+ return response
+
+ def build_simple_metrics_response(self, record):
+ single_result = dict()
+ single_result["id"] = record[0]
+ single_result["estimator_id"] = record[1]
+ single_result["TP"] = record[2]
+ single_result["FP"] = record[3]
+ single_result["TN"] = record[4]
+ single_result["FN"] = record[5]
+ single_result["sensitivity"] = record[6]
+ single_result["specificity"] = record[7]
+ single_result["accuracy"] = record[8]
+ single_result["balanced_accuracy"] = record[9]
+ single_result["precision"] = record[10]
+ single_result["recall"] = record[11]
+ single_result["PPV"] = record[12]
+ single_result["NPV"] = record[13]
+ single_result["FNR"] = record[14]
+ single_result["FPR"] = record[15]
+ single_result["FDR"] = record[16]
+ single_result["FOR"] = record[17]
+ single_result["f1"] = record[18]
+ single_result["f0.5"] = record[19]
+ single_result["f2"] = record[20]
+ single_result["MCC"] = record[21]
+ single_result["ROCAUC"] = record[22]
+ single_result["Youdens_statistic"] = record[23]
+ return single_result
+
+ def get_all_estimator(self):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.estimator"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ response = list()
+ if query_result:
+ response = self.build_estimator_response(query_result)
+ return response
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+ single_response = self.build_complex_metrics_response(query_result)
+ return single_response
+
+ def get_estimator_by_train_task_id(self, train_task_id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT * FROM common_fraud.estimator WHERE train_task_id = %s"
+ parameter = (train_task_id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ response = list()
+ if query_result:
+ response = self.build_estimator_response(query_result)
+ return response
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def build_estimator_response(self, query_result):
+ summarized_result = list()
+ for item in query_result:
+ single_result = dict()
+ single_result["id"] = item[0]
+ single_result["train_task_id"] = item[1]
+ single_result["transaction_identifier_field_name"] = item[2]
+ summarized_result.append(single_result)
+ return summarized_result
+
+ def get_transaction_identifier_field_name_by_estimator_id(self, id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT transaction_identifier_field_name FROM common_fraud.estimator WHERE id = %s"
+ parameter = (id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ return query_result[0] if query_result else None
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_confusion_matrix_elements_by_train_task_id(self, train_task_id):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = "SELECT TP, TN, FP, FN FROM common_fraud.metrics inner join common_fraud.estimator e on metrics.estimator_id = e.id inner join common_fraud.train_task tt on e.train_task_id = tt.id where train_task_id = %s"
+ parameter = (train_task_id,)
+ try:
+ cursor.execute(sql_select_query, parameter)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ response = dict()
+ if query_result:
+ response["TP"] = query_result[0]
+ response["TN"] = query_result[1]
+ response["FP"] = query_result[2]
+ response["FN"] = query_result[3]
+ return response
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_columns_contain_only_distinct_values(self, schema_name, table_name):
+ result = list()
+ column_names = self.get_field_name_in_ordinal_position(schema_name, table_name)
+ count_by_column_name = dict()
+ for column_name in column_names:
+ count = self.get_distinct_count_of_column(schema_name, table_name, column_name)
+ count_by_column_name[column_name] = count
+ detailed_info = self.get_detailed_information_about_table(schema_name, table_name)
+ record_number = detailed_info.get("record_number")
+ for column_name, count in count_by_column_name.items():
+ if count == record_number:
+ result.append(column_name)
+ return result
+
+ def get_distinct_count_of_column(self, schema_name, table_name, column_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT COUNT(DISTINCT {column_name}) FROM {schema_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ count = query_result[0]
+ self.logger.debug(
+ f"Distinct count of values: {count} in {schema_name} schema, {table_name} table: {column_name} column.")
+ return count
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def create_reduced_and_for_evaluation_tables(self, schema_name, table_name, fraud_type_field_name,
+ positive_fraud_case_number, negative_fraud_case_number,
+ date_suffix_to_append):
+ detailed_information_about_table = self.get_detailed_information_about_table(schema_name, table_name)
+ fraud_candidates = detailed_information_about_table.get("fraud_candidates")
+ fraud_candidate_names = list()
+ for fraud_candidate in fraud_candidates:
+ fraud_candidate_name = fraud_candidate.get("name")
+ fraud_candidate_names.append(fraud_candidate_name)
+ if fraud_type_field_name not in fraud_candidate_names:
+ raise IllegalArgumentException({"message": "Fraud type field name not in fraud candidates",
+ "parameter_name": fraud_type_field_name})
+ positive_fraud_case_ids = self.get_id_collection_by_column_name_and_value(schema_name, table_name,
+ fraud_type_field_name, 1)
+ negative_fraud_case_ids = self.get_id_collection_by_column_name_and_value(schema_name, table_name,
+ fraud_type_field_name, 0)
+ random_selected_positive_fraud_case_ids = random.sample(positive_fraud_case_ids, k=positive_fraud_case_number)
+ random_selected_negative_fraud_case_ids = random.sample(negative_fraud_case_ids, k=negative_fraud_case_number)
+ records_by_id = self.get_records(schema_name, table_name)
+ field_names_in_ordinal_position = self.get_field_name_in_ordinal_position(schema_name, table_name)
+ fraud_field_index = field_names_in_ordinal_position.index(fraud_type_field_name) - 1
+ reduced_records = list()
+ for_evaluation_records = list()
+ simulated_real_recods = list()
+ for id, record in records_by_id.items():
+ if id not in random_selected_positive_fraud_case_ids and id not in random_selected_negative_fraud_case_ids:
+ reduced_records.append(record)
+ for id, record in records_by_id.items():
+ if id in random_selected_negative_fraud_case_ids:
+ fraud_field_value = record.pop(fraud_field_index)
+ for_evaluation_record = [id]
+ for_evaluation_record.extend(record)
+ for_evaluation_records.append(for_evaluation_record)
+ simulated_real_recods.append([id, fraud_field_value])
+ for id, record in records_by_id.items():
+ if id in random_selected_positive_fraud_case_ids:
+ fraud_field_value = record.pop(fraud_field_index)
+ for_evaluation_record = [id]
+ for_evaluation_record.extend(record)
+ for_evaluation_records.append(for_evaluation_record)
+ simulated_real_recods.append([id, fraud_field_value])
+ field_name_and_type_collection = detailed_information_about_table.get("fields")
+ primary_key_field_name = detailed_information_about_table.get("primary_key")
+ field_type_by_name_for_evaluation = dict()
+ field_type_by_name_reduced = dict()
+ field_type_by_name_simulated_real = dict()
+ for item in field_name_and_type_collection:
+ field_name = item.get("name")
+ field_type = item.get("type")
+ if field_name == primary_key_field_name:
+ field_type_by_name_simulated_real["transaction_identifier"] = field_type
+ else:
+ if field_type == "varchar":
+ field_type += "(255)"
+ if field_name == fraud_type_field_name:
+ field_type_by_name_reduced[field_name] = field_type
+ field_type_by_name_simulated_real[field_name] = field_type
+ else:
+ field_type_by_name_reduced[field_name] = field_type
+ field_type_by_name_for_evaluation[field_name] = field_type
+ reduced_table_name = table_name + "_reduced"
+ for_evaluation_table_name = table_name + "_for_evaluation"
+ simulated_real_table_name = table_name + "_simulated_real"
+ if date_suffix_to_append:
+ now = datetime.datetime.now()
+ date_and_time = now.strftime("%Y%m%d_%H%M%S")
+ date_time_suffix = UNDER_SCORE + str(date_and_time)
+ reduced_table_name += date_time_suffix
+ for_evaluation_table_name += date_time_suffix
+ simulated_real_table_name += date_time_suffix
+ self.create_train_table(schema_name, reduced_table_name, field_type_by_name_reduced,
+ is_id_auto_incremented=True)
+ self.create_train_table(schema_name, for_evaluation_table_name, field_type_by_name_for_evaluation,
+ is_id_auto_incremented=False)
+ self.create_train_table(schema_name, simulated_real_table_name, field_type_by_name_simulated_real,
+ is_id_auto_incremented=True)
+
+ reduced_records_array = np.array(reduced_records)
+ for_evaluated_records_array = np.array(for_evaluation_records)
+ simulated_real_records_array = np.array(simulated_real_recods,dtype='O')
+ ddl_command_builder = ddl_build_module.DdlCommandBuilder()
+ reduced_records_insert_script = ddl_command_builder.create_insert_into_generic_train_database_script(
+ schema_name, reduced_table_name, field_type_by_name_reduced, is_id_auto_incremented=True)
+ for_evaluated_records_insert_script = ddl_command_builder.create_insert_into_generic_train_database_script(
+ schema_name, for_evaluation_table_name, field_type_by_name_for_evaluation, is_id_auto_incremented=False)
+ simulated_real_recods_insert_script = ddl_command_builder.create_insert_into_generic_train_database_script(
+ schema_name, simulated_real_table_name, field_type_by_name_simulated_real, is_id_auto_incremented=True)
+
+ self.persist_encoded_or_engineered_dataset(reduced_records_insert_script, reduced_records_array, REDUCED)
+ self.persist_encoded_or_engineered_dataset(for_evaluated_records_insert_script, for_evaluated_records_array,
+ FOR_EVAULATED)
+ self.persist_encoded_or_engineered_dataset(simulated_real_recods_insert_script, simulated_real_records_array,
+ SIMULATED_REAL)
+ planned_csv_content = dict()
+ for record in simulated_real_recods:
+ planned_csv_content[record[0]] = record[1]
+ csv_content_and_file_name = dict()
+ csv_content_and_file_name["planned_csv_content"] = planned_csv_content
+ csv_content_and_file_name["planned_csv_file_name"] = for_evaluation_table_name
+ return csv_content_and_file_name
+
+ def get_id_collection_by_column_name_and_value(self, schema_name, table_name, column_name, value):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT id FROM {schema_name}.{table_name} WHERE {column_name} = {value}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchall()
+ cursor.close()
+ connection_object.close()
+ result = list()
+ for item in query_result:
+ result.append(item[0])
+ return result
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
+
+ def get_generic_records_for_evaluation(self, schema_name, table_name):
+ field_names = self.get_field_name_in_ordinal_position(schema_name, table_name)
+ field_types = self.get_field_type_in_ordinal_position(schema_name, table_name)
+ detailed_information = self.get_detailed_information_about_table(schema_name, table_name)
+ primary_key_field_name = detailed_information.get("primary_key")
+ record_by_id = self.get_records(schema_name, table_name)
+ records_in_string_format = list()
+ for id, record in record_by_id.items():
+ fields_as_str = list()
+ fields_as_str.append(id)
+ for field in record:
+ fields_as_str.append(str(field))
+ records_in_string_format.append(fields_as_str)
+ result = dict()
+ result["field_types"] = field_types
+ result["field_names"] = field_names
+ result["records"] = records_in_string_format
+ return result
+
+ def get_max_retrospective_day(self,schema_name,table_name,time_base_field_name):
+ connection_object = self.connection_pool.get_connection()
+ cursor = connection_object.cursor()
+ sql_select_query = f"SELECT DATEDIFF(max({time_base_field_name}),min({time_base_field_name}))id FROM {schema_name}.{table_name}"
+ try:
+ cursor.execute(sql_select_query)
+ query_result = cursor.fetchone()
+ cursor.close()
+ connection_object.close()
+ return query_result[0]
+ except mysql.connector.Error as err:
+ self.logger.error(f"MySQL error message: {err.msg}")
\ No newline at end of file
--- /dev/null
+import logging
+
+import exception_module
+import re
+
+COMMA = ","
+SPACE = " "
+NEW_LINE = "\n"
+BEGIN_BRACKET = "("
+END_BRACKET = ")"
+DOT = "."
+
+
+class DdlCommandBuilder:
+ """DDL"""
+ def __init__(self):
+ self.logger = logging.getLogger("train.server.database.ddl")
+
+ def build_create_encoded_or_engineered_table_script(self, schema_name, table_name, field_names):
+ script = "CREATE TABLE IF NOT EXISTS" + SPACE + schema_name + DOT + table_name
+ script += BEGIN_BRACKET
+ script += "id BIGINT NOT NULL AUTO_INCREMENT" + COMMA
+ for field_name in field_names:
+ script += field_name + SPACE + "DOUBLE PRECISION" + COMMA
+ script += "PRIMARY KEY (id)"
+ script += END_BRACKET + SPACE
+ script += "ENGINE = InnoDB"
+ return script
+
+ def create_insert_into_encoded_or_feature_engineered_script(self, schema_name, table_name, field_names):
+ script = "INSERT INTO" + SPACE + schema_name + DOT + table_name + SPACE
+ script += BEGIN_BRACKET
+ field_number = len(field_names)
+ for i in range(field_number):
+ script += field_names[i]
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET + SPACE
+ script += "VALUES"
+ script += BEGIN_BRACKET
+ for i in range(field_number):
+ script += "%s"
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET
+ return script
+
+ def create_insert_into_encoded_or_feature_engineered_script_for_prediction(self, schema_name, table_name,
+ field_names,
+ fraud_type_field_name):
+ script = "INSERT INTO" + SPACE + schema_name + DOT + table_name + SPACE
+ script += BEGIN_BRACKET
+ field_names.remove(fraud_type_field_name)
+ field_number = len(field_names)
+ for i in range(field_number):
+ script += field_names[i]
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET + SPACE
+ script += "VALUES"
+ script += BEGIN_BRACKET
+ for i in range(field_number):
+ script += "%s"
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET
+ return script
+
+ def build_create_generic_train_database_script(self, schema_name, table_name, field_type_by_name,
+ is_id_auto_incremented):
+ possible_field_types_without_varchar = ["BIGINT", "INT", "DOUBLE PRECISION", "FLOAT", "DATE", "TIME",
+ "DATETIME", "TIMESTAMP", "YEAR"]
+ script = "CREATE TABLE IF NOT EXISTS" + SPACE + schema_name + DOT + table_name
+ script += BEGIN_BRACKET
+ script += "id BIGINT NOT NULL"
+ script += SPACE
+ if is_id_auto_incremented:
+ script += "AUTO_INCREMENT"
+ script += COMMA
+ for field_name, field_type in field_type_by_name.items():
+ upper_case_field_type = field_type.upper().strip()
+ is_regular_varchar = re.search("^VARCHAR\([0-9]+\)", upper_case_field_type)
+ if upper_case_field_type not in possible_field_types_without_varchar and is_regular_varchar is None:
+ raise exception_module.DdlBuildException(
+ f"invalid field type, field type name: {upper_case_field_type}")
+
+ if upper_case_field_type.startswith("VARCHAR"):
+ try:
+ begin_bracket_index = upper_case_field_type.index("(")
+ end_brack_index = upper_case_field_type.index(")")
+ varchar_length_as_string = upper_case_field_type[begin_bracket_index + 1:end_brack_index]
+ int(varchar_length_as_string)
+ except ValueError as e:
+ raise exception_module.DdlBuildException("varchar size parameter isn't int type")
+ if upper_case_field_type.startswith("DATETIME"):
+ upper_case_field_type += "(6)"
+ script += field_name + SPACE + upper_case_field_type + COMMA
+ script += "PRIMARY KEY (id)"
+ script += END_BRACKET + SPACE
+ script += "ENGINE = InnoDB"
+ return script
+
+ def create_insert_into_generic_train_database_script(self, schema_name, table_name, field_type_by_name,
+ is_id_auto_incremented):
+ script = "INSERT INTO" + SPACE + schema_name + DOT + table_name + SPACE
+ script += BEGIN_BRACKET
+ if not is_id_auto_incremented:
+ script += "id"
+ script += COMMA
+ field_names = list(field_type_by_name.keys())
+ field_number = len(field_names)
+ for i in range(field_number):
+ script += field_names[i]
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET + SPACE
+ script += "VALUES"
+ script += BEGIN_BRACKET
+ if not is_id_auto_incremented:
+ script += "%s"
+ script += COMMA
+ for i in range(field_number):
+ script += "%s"
+ if i != field_number - 1:
+ script += COMMA
+ script += END_BRACKET
+ return script
--- /dev/null
+import logging
+import pickle
+
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils import column_or_1d
+
+PRIMARY_KEY = "primary_key"
+OMIT = "omit"
+FRAUD_TYPE = "fraud_type"
+JULIAN = "julian"
+LABEL_ENCODER = "label_encoder"
+FLOAT = "float"
+INT = "int"
+
+
+class NotImplementedEncodingException(Exception):
+ pass
+
+
+class DataBaseEncoder:
+ def __init__(self, dataset, original_fields):
+ self.logger = logging.getLogger("train.server.database.encoding")
+ self.original_dataset = dataset
+ self.original_fields = original_fields
+
+ def encode(self, encoding_parameters, detailed_information_about_table):
+ primary_key_field_name = detailed_information_about_table.get(PRIMARY_KEY)
+ encoder_by_field_name = dict()
+ encoded_fields = self.build_encoded_field_names(encoding_parameters, primary_key_field_name)
+ number_of_records = len(self.original_dataset)
+ number_of_fields = len(encoded_fields)
+ encoded_array = np.empty([number_of_records, number_of_fields])
+ for field in encoded_fields:
+ field_type_before_encoding = self.get_field_type_before_encoding(field, detailed_information_about_table)
+ column_index_in_original_dataset = self.original_fields.index(field)
+ column_index_in_encoded_array = encoded_fields.index(field)
+ encoding_parameter = encoding_parameters.get(field)
+ if encoding_parameter is None:
+ if self.is_num_type(field_type_before_encoding):
+ encoded_array[:,
+ column_index_in_encoded_array: column_index_in_encoded_array + 1] = self.original_dataset[:,
+ column_index_in_original_dataset: column_index_in_original_dataset + 1]
+ continue
+ elif encoding_parameter == FRAUD_TYPE:
+ encoded_array[:,
+ column_index_in_encoded_array: column_index_in_encoded_array + 1] = self.original_dataset[:,
+ column_index_in_original_dataset: column_index_in_original_dataset + 1]
+ continue
+ elif encoding_parameter == JULIAN:
+ encoded_array[:,
+ column_index_in_encoded_array: column_index_in_encoded_array + 1] = self.convert_timestamp_to_julian(column_index_in_original_dataset)
+ continue
+ elif encoding_parameter == LABEL_ENCODER:
+ converted_column, encoder = self.convert_with_label_encoder(column_index_in_original_dataset)
+ encoded_array[:, column_index_in_encoded_array: column_index_in_encoded_array + 1] = converted_column
+ encoder_by_field_name[field] = encoder
+ continue
+ elif encoding_parameter == FLOAT:
+ encoded_array[:,
+ column_index_in_encoded_array: column_index_in_encoded_array + 1] = self.convert_string_to_float(column_index_in_original_dataset)
+ continue
+ elif encoding_parameter == INT:
+ encoded_array[:,
+ column_index_in_encoded_array: column_index_in_encoded_array + 1] = self.convert_string_to_int(column_index_in_original_dataset)
+ continue
+ else:
+ raise NotImplementedEncodingException(
+ {"message": "The given encoding not implemented", "field_name": field,
+ "encoding type": encoding_parameter})
+ return encoded_array, encoder_by_field_name, encoded_fields
+
+ # 3.1.5.1
+ def build_encoded_field_names(self, encoding_parameters, primary_key_field_name):
+ feature_fields = list()
+ fraud_type_field = None
+ number_of_original_fields = len(self.original_fields)
+ for index in range(number_of_original_fields):
+ field = self.original_fields[index]
+ if field == primary_key_field_name:
+ continue # id-re nincs szükség auto-increment miatt
+ elif encoding_parameters.get(field) == FRAUD_TYPE:
+ fraud_type_field = field
+ continue
+ elif encoding_parameters.get(field) == OMIT:
+ continue # elhagyandó
+ feature_fields.append(field)
+ if fraud_type_field is not None:
+ feature_fields.append(fraud_type_field)
+ return feature_fields
+
+ def get_field_type_before_encoding(self, field, detailed_information_about_table):
+ result = None
+ for field_property in detailed_information_about_table.get("fields"):
+ if field == field_property.get("name"):
+ result = field_property.get("type")
+ return result
+
+ def is_num_type(self, item):
+ integer_types = ("integer", "int", "smallint")
+ float_types = ("float", "double")
+ if item in integer_types or item in float_types:
+ return True
+ else:
+ return False
+
+ def convert_timestamp_to_julian(self, original_index):
+ converted_time_stamp_datas = list()
+ for timestamp in self.original_dataset[:, original_index:original_index + 1]:
+ t = timestamp[0]
+ ts = pd.Timestamp(t)
+ converted_time_stamp_to_julian = ts.to_julian_date()
+ converted_time_stamp_datas.append(converted_time_stamp_to_julian)
+ converted_time_stamp_data_array = np.array(converted_time_stamp_datas)
+ reshaped_array = converted_time_stamp_data_array.reshape(-1, 1)
+ return reshaped_array
+
+ def convert_with_label_encoder(self, original_index):
+ strings = self.original_dataset[:, original_index:original_index + 1]
+ encoder = LabelEncoder()
+ modified_strings = column_or_1d(strings)
+ encoder.fit(modified_strings)
+ transformed_array = encoder.transform(modified_strings)
+ reshaped_transformed_array = transformed_array.reshape(-1, 1)
+ return reshaped_transformed_array, encoder
+
+ def convert_string_to_float(self, original_index):
+ strings = self.original_dataset[:, original_index:original_index + 1]
+ floats = np.array(strings, dtype=float)
+ reshaped_array = floats.reshape(-1, 1)
+ return reshaped_array
+
+ def convert_string_to_int(self, original_index):
+ strings = self.original_dataset[:, original_index:original_index + 1]
+ ints = np.array(strings, dtype=int)
+ reshaped_array = ints.reshape(-1, 1)
+ return reshaped_array
+
+ # új metódusok vége
+
+ def convertCardNumberStringToFloat(self, array):
+ cardNumberStrings = array[:, 1:2]
+ # cardEncoder=LabelEncoder()
+ # modifiedCardNumbers=column_or_1d(cardNumberStrings)
+ # cardEncoder.fit(modifiedCardNumbers)
+ # encodedCardNumbers=cardEncoder.transform(modifiedCardNumbers)
+ # reshapedCardNumbers=encodedCardNumbers.reshape(-1,1)
+ cardNumberFloats = np.array(cardNumberStrings, dtype=float)
+ reshapedCardNumberIntegers = cardNumberFloats.reshape(-1, 1)
+ array[:, 1:2] = reshapedCardNumberIntegers
+ # array[:,1:2]=reshapedCardNumbers
+
+ def convertVendorCodeStringToFloat(self, array):
+ vendorCodeStrings = array[:, 8:9]
+ vendorCodeIntegers_as_int_type = vendorCodeStrings.astype(np.int)
+ vendorCodeIntegers_as_float_type = vendorCodeStrings.astype(np.float)
+ vendorCodeIntegers = np.array(vendorCodeStrings, dtype=int)
+ reshapedVendorCodeIntegers = vendorCodeIntegers.reshape(-1, 1)
+ array[:, 8:9] = reshapedVendorCodeIntegers
+
+ def convertCountryFeature(self, array):
+ countries = array[:, 7:8]
+ countryEncoder = LabelEncoder()
+ modifiedCountries = column_or_1d(countries)
+ countryEncoder.fit(modifiedCountries)
+ encodedCountries = countryEncoder.transform(modifiedCountries)
+ reshapedEncodedCountries = encodedCountries.reshape(-1, 1)
+ array[:, 7:8] = reshapedEncodedCountries
+ return countryEncoder
+
+ def convertCurrencyFeature(self, array):
+ currenciesArray = array[:, 5:6]
+ currencyEncoder = LabelEncoder()
+ currencyEncoder.fit(currenciesArray)
+ encodedCurrencies = currencyEncoder.transform(currenciesArray)
+ reshapedEncodedCurrencies = encodedCurrencies.reshape(-1, 1)
+ array[:, 5:6] = reshapedEncodedCurrencies
+ return currencyEncoder
+
+ def saveData(self, dataSet, connection, dataBaseName):
+ valuesArray = dataSet[:, 1:]
+ cursor = connection.cursor()
+ sqlUseQuery = "USE " + dataBaseName
+ cursor.execute(sqlUseQuery)
+ sqlInsertQuery = "INSERt INTO encoded_transaction (card_number,transaction_type,timestamp,amount,currency_name,response_code,country_name,vendor_code,fraud) VALUES " \
+ "(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
+ length = len(valuesArray)
+ bound = 1000
+ if length > bound:
+ numberOfPartArray = int(length / bound)
+ numberOfRestDatas = length - numberOfPartArray * bound
+ for i in range(0, numberOfPartArray, 1):
+ tempArray = valuesArray[i * bound:(i + 1) * bound, :]
+ valueList = list()
+ for record in tempArray:
+ valueList.append(tuple(record))
+ cursor.executemany(sqlInsertQuery, valueList)
+ connection.commit()
+ tempArray = valuesArray[(numberOfPartArray) * bound:(numberOfPartArray) * bound + numberOfRestDatas, :]
+ valueList = list()
+ for record in tempArray:
+ valueList.append(tuple(record))
+ cursor.executemany(sqlInsertQuery, valueList)
+ connection.commit()
+ else:
+ valueList = list()
+ for record in valuesArray:
+ valueList.append(tuple(record))
+ cursor.executemany(sqlInsertQuery, valueList)
+ connection.commit()
+ cursor.close()
+
+ def saveEncoder(self, connection, databaseName):
+ cursor = connection.cursor()
+ sqlUseQuery = "USE " + databaseName
+ cursor.execute(sqlUseQuery)
+ sqlInsertQuery = "INSERT INTO encoder (encoder_name,encoder_object) VALUES (%s,%s)"
+ pickledCurrencyEncoder = pickle.dumps(self.currencyEncoder)
+ cursor.execute(sqlInsertQuery, ("currency_encoder", pickledCurrencyEncoder))
+ pickledCountryEncoder = pickle.dumps(self.countryEncoder)
+ cursor.execute(sqlInsertQuery, ("country_encoder", pickledCountryEncoder))
+ connection.commit()
+ cursor.close()
--- /dev/null
+import logging
+import math
+import statistics
+import multiprocessing as mp
+from multiprocessing import Process, Pool
+import numpy as np
+
+CARD_NUMBER = "card_number"
+UNDER_SCORE = "_"
+
+
+class Engineer:
+ def __init__(self, time_base_field_name, encoded_field_names, fraud_type_field_name,feature_engineering_parameters, used_cpu_core):
+ self.logger = logging.getLogger("train.server.database.engineer")
+ self.time_base_field_name = time_base_field_name
+ self.encoded_field_names = encoded_field_names
+ self.fraud_type_field_name=fraud_type_field_name
+ self.feature_engineering_parameters = feature_engineering_parameters
+ self.chosen_feature_field_name = self.feature_engineering_parameters.get("chosen_feature_field")
+ self.intervals = self.feature_engineering_parameters.get("intervals")
+ self.used_cpu_core = used_cpu_core
+
+ # 3.1.7
+ def create_new_features(self, dataset):
+ self.logger.info(f"Used CPU logical core number: {self.used_cpu_core}")
+ feature_engineered_field_names = self.crete_feature_engineered_field_names(self.encoded_field_names,
+ self.chosen_feature_field_name,
+ self.intervals)
+ transaction_by_card_number = self.get_transaction_by_card_number(dataset, self.encoded_field_names)
+ input_collection = list()
+ for card_number in transaction_by_card_number:
+ input_collection.append(transaction_by_card_number.get(card_number))
+ results_from_threads = list()
+ with Pool(processes=self.used_cpu_core) as pool:
+ results_from_threads = pool.map(self.process_dataset_of_single_cardnumber, input_collection)
+ self.logger.info(f"Number of sublists from threads: {len(results_from_threads)}")
+ num_of_sub_results=len(results_from_threads)
+ concatenated_results=np.empty([0,0])
+ for i in range(num_of_sub_results):
+ if i==0:
+ concatenated_results=results_from_threads[i]
+ else:
+ concatenated_results=np.concatenate((concatenated_results,results_from_threads[i]),axis=0)
+
+ return concatenated_results, feature_engineered_field_names
+
+ def crete_feature_engineered_field_names(self, field_names, chosen_feature_field_name, time_intervals):
+ result = list()
+ real_fraud_field_name=None
+ for field_name in field_names:
+ if field_name==self.fraud_type_field_name:
+ real_fraud_field_name=field_name
+ else:
+ result.append(field_name)
+ for interval in time_intervals:
+ result.append(chosen_feature_field_name + "_current_per_average_" + str(interval))
+ result.append(chosen_feature_field_name + "_current_minus_average_" + str(interval))
+ result.append(chosen_feature_field_name + "_current_per_median_" + str(interval))
+ result.append(chosen_feature_field_name + "_current_minus_median_" + str(interval))
+ result.append(
+ chosen_feature_field_name + "_current_minus_average_" + str(interval) + "_per_deviation_" + str(
+ interval))
+ result.append(
+ chosen_feature_field_name + "_current_minus_average_" + str(interval) + "_minus_deviation_" + str(
+ interval))
+ result.append("tr_nr_per_avg_nr_" + str(interval))
+ result.append("tr_nr_minus_avg_nr_" + str(interval))
+ result.append("tr_nr_per_median_nr_" + str(interval))
+ result.append("tr_nr_minus_median_nr_" + str(interval))
+ result.append("avg_tr_interval_per_tr_interval_" + str(interval))
+ result.append("avg_tr_interval_minus_tr_interval_" + str(interval))
+ result.append("median_tr_interval_per_tr_interval_" + str(interval))
+ result.append("median_tr_interval_minus_tr_interval_" + str(interval))
+ result.append(real_fraud_field_name)
+ return result
+
+ def get_transaction_by_card_number(self, dataset, encoded_field_names):
+ result = dict()
+ for record in dataset:
+ index = encoded_field_names.index(CARD_NUMBER)
+ card_number = record[index]
+ if result.get(card_number) is None:
+ transactions = list()
+ transactions.append(record.tolist())
+ result[card_number] = transactions
+ else:
+ transactions = result.get(card_number)
+ transactions.append(record.tolist())
+ return result
+
+ def process_dataset_of_single_cardnumber(self, input_data_of_given_card_number):
+ number_of_new_feature = len(self.intervals) * 14
+ extended_dataset = list()
+ data_set = np.array(input_data_of_given_card_number)
+ transaction_features = data_set[:, :-1]
+ transaction_labels = data_set[:, -1:]
+ length = len(transaction_features)
+ for i in range(length):
+ transaction_feature = transaction_features[i]
+ index_of_feature_to_be_engineered = self.encoded_field_names.index(self.chosen_feature_field_name)
+ feature_value = transaction_feature[index_of_feature_to_be_engineered]
+ index_of_time_stamp = self.encoded_field_names.index(self.time_base_field_name)
+ current_time_stamp = transaction_feature[index_of_time_stamp]
+ transaction_feature_list = list(transaction_feature)
+
+ if i == length - 1:
+ for j in range(number_of_new_feature):
+ transaction_feature_list.append(0)
+ extended_dataset.append(transaction_feature_list)
+ else:
+ feature_to_be_engineered_by_time_stamp, time_stamps = self.get_feature_to_be_engineered_by_timestamp_before_given_time_stamp(
+ transaction_features, current_time_stamp, index_of_feature_to_be_engineered, index_of_time_stamp)
+
+ generated_features = list()
+
+ generated_features_based_on_chosen_feature = self.get_generated_features_based_on_chosen_feature(
+ current_time_stamp, feature_value, feature_to_be_engineered_by_time_stamp)
+
+ generated_features_based_on_transaction_number = self.get_generated_features_based_on_transaction_number(
+ current_time_stamp, time_stamps)
+
+ generated_features_based_on_transaction_interval = self.get_generated_features_based_on_transaction_interval(
+ current_time_stamp, time_stamps)
+
+ generated_features.extend(generated_features_based_on_chosen_feature)
+ generated_features.extend(generated_features_based_on_transaction_number)
+ generated_features.extend(generated_features_based_on_transaction_interval)
+ number_of_generated_features = len(generated_features)
+ for interval in self.intervals:
+ for k in range(number_of_generated_features):
+ transaction_feature_list.append(generated_features[k].get(interval))
+
+ label = transaction_labels[i][0]
+ transaction_feature_list.append(label)
+ transaction_feature_tuple = tuple(transaction_feature_list)
+ extended_dataset.append(transaction_feature_tuple)
+ result=np.array(extended_dataset)
+ return result
+
+ def get_feature_to_be_engineered_by_timestamp_before_given_time_stamp(self, data_set, given_time_stamp,
+ index_of_feature_to_be_engineered,
+ index_of_timestamp):
+ feature_by_timestamp = dict()
+ time_stamps = list()
+ for record in data_set:
+ current_time_stamp_value = record[index_of_timestamp]
+ feature_to_be_engineered_value = record[index_of_feature_to_be_engineered]
+ if current_time_stamp_value < given_time_stamp:
+ time_stamps.append(current_time_stamp_value)
+ feature_by_timestamp[current_time_stamp_value] = feature_to_be_engineered_value
+ return feature_by_timestamp, time_stamps
+
+ def get_generated_features_based_on_chosen_feature(self, current_timestamp, feature_value,
+ feature_to_be_engineered_by_timestamp):
+
+ feature_values_by_retrospective_time = dict()
+ for interval in self.intervals:
+ feature_values = list()
+ for timestamp_in_julian_date, feature_value in feature_to_be_engineered_by_timestamp.items():
+ if timestamp_in_julian_date > current_timestamp - interval:
+ feature_values.append(feature_value)
+ feature_values_by_retrospective_time[interval] = feature_values
+
+ average_feature_value_by_retrospective_time = dict()
+ deviation_feature_value_by_retrospective_time = dict()
+ median_feature_value_by_retrospective_time = dict()
+
+ for interval in self.intervals:
+ feature_values = feature_values_by_retrospective_time.get(interval)
+ try:
+ average_feature_value_by_retrospective_time[interval] = statistics.mean(
+ feature_values) if feature_values else 0
+ median_feature_value_by_retrospective_time[interval] = statistics.median(
+ feature_values) if feature_values else 0
+ deviation_feature_value_by_retrospective_time[interval] = statistics.stdev(feature_values) if len(
+ feature_values) >= 2 else 0
+ except TypeError as e:
+ self.logger.error("error")
+
+ feature_per_average_feature_value_by_retrospective_time = dict()
+ feature_minus_average_feature_value_by_retrospective_time = dict()
+ feature_per_median_feature_value_by_retrospective_time = dict()
+ feature_minus_median_feature_value_by_retrospective_time = dict()
+ feature_minus_average_feature_per_deviation_feature_value_by_retrospective_time = dict()
+ feature_minus_average_feature_minus_deviation_feature_value_by_retrospective_time = dict()
+
+ for interval in self.intervals:
+ feature_per_average_feature_value_by_retrospective_time[
+ interval] = feature_value / average_feature_value_by_retrospective_time.get(
+ interval) if average_feature_value_by_retrospective_time.get(interval) != 0 else 0
+ feature_minus_average_feature_value_by_retrospective_time[
+ interval] = feature_value - average_feature_value_by_retrospective_time.get(interval)
+ feature_per_median_feature_value_by_retrospective_time[
+ interval] = feature_value / median_feature_value_by_retrospective_time.get(
+ interval) if median_feature_value_by_retrospective_time.get(interval) != 0 else 0
+ feature_minus_median_feature_value_by_retrospective_time[
+ interval] = feature_value - median_feature_value_by_retrospective_time.get(interval)
+ feature_minus_average_feature_per_deviation_feature_value_by_retrospective_time[
+ interval] = feature_value / deviation_feature_value_by_retrospective_time.get(
+ interval) if deviation_feature_value_by_retrospective_time.get(interval) != 0 else 0
+ feature_minus_average_feature_minus_deviation_feature_value_by_retrospective_time[
+ interval] = feature_value - deviation_feature_value_by_retrospective_time.get(interval)
+ return [feature_per_average_feature_value_by_retrospective_time,
+ feature_minus_average_feature_value_by_retrospective_time,
+ feature_per_median_feature_value_by_retrospective_time,
+ feature_minus_median_feature_value_by_retrospective_time,
+ feature_minus_average_feature_per_deviation_feature_value_by_retrospective_time,
+ feature_minus_average_feature_minus_deviation_feature_value_by_retrospective_time]
+
+ def get_feature_values_by_julian_date(self, feature_and_timestamp_values):
+ feature_values_by_date_dictionary = dict()
+ for current_feature_and_timestamp in feature_and_timestamp_values:
+ current_feature = current_feature_and_timestamp[0]
+ timestamp = current_feature_and_timestamp[1]
+ current_julian_date = int(timestamp)
+ if feature_values_by_date_dictionary.get(current_julian_date) is None:
+ feature_collection = list()
+ feature_collection.append(current_feature)
+ feature_values_by_date_dictionary[current_julian_date] = feature_collection
+ else:
+ feature_collection = feature_values_by_date_dictionary.get(current_julian_date)
+ feature_collection.append(current_feature)
+ return feature_values_by_date_dictionary
+
+ def get_generated_features_based_on_transaction_number(self, current_time_stamp,
+ time_stamps):
+
+ transaction_number_on_current_day = 1
+ for time_stamp_in_julian_format in time_stamps:
+ if time_stamp_in_julian_format > current_time_stamp - 1:
+ transaction_number_on_current_day += 1
+
+ transaction_number_by_interval = dict()
+ for interval in self.intervals:
+ transaction_number = 0
+ transaction_number_by_day_order_number = dict()
+ for i in range(interval):
+ transaction_number = 0
+ upper_boundara = current_time_stamp - i
+ lower_boundary = current_time_stamp - (i + 1)
+ for time_stamp_in_julian_format in time_stamps:
+ if time_stamp_in_julian_format < upper_boundara and time_stamp_in_julian_format > lower_boundary:
+ transaction_number += 1
+ transaction_number_by_day_order_number[i] = transaction_number
+ transactions_on_days = list()
+ for transaction_number_on_given_day in transaction_number_by_day_order_number.values():
+ transactions_on_days.append(transaction_number_on_given_day)
+ transaction_number_by_interval[interval] = transactions_on_days
+
+ daily_average_transaction_number_by_retrospective_time = dict()
+ daily_median_transaction_number_by_retrospective_time = dict()
+
+ for interval in self.intervals:
+ transaction_number = transaction_number_by_interval.get(interval)
+ daily_average_transaction_number_by_retrospective_time[interval] = statistics.mean(
+ transaction_number) if transaction_number else 0
+ daily_median_transaction_number_by_retrospective_time[interval] = statistics.median(
+ transaction_number) if transaction_number else 0
+
+ transaction_number_on_current_day_per_daily_average_transaction_number_by_retrospective_time = dict()
+ transaction_number_on_current_day_minus_daily_average_transaction_number_by_retrospective_time = dict()
+ transaction_number_on_current_day_per_daily_median_transaction_number_by_retrospective_time = dict()
+ transaction_number_on_current_day_minus_daily_median_transaction_number_by_retrospective_time = dict()
+ for interval in self.intervals:
+ average_transaction_number = daily_average_transaction_number_by_retrospective_time.get(interval)
+ median_transaction_number = daily_median_transaction_number_by_retrospective_time.get(interval)
+ transaction_number_on_current_day_per_daily_average_transaction_number_by_retrospective_time[
+ interval] = transaction_number_on_current_day / average_transaction_number if average_transaction_number != 0 else 0
+ transaction_number_on_current_day_minus_daily_average_transaction_number_by_retrospective_time[
+ interval] = transaction_number_on_current_day - average_transaction_number
+ transaction_number_on_current_day_per_daily_median_transaction_number_by_retrospective_time[
+ interval] = transaction_number_on_current_day / median_transaction_number if median_transaction_number != 0 else 0
+ transaction_number_on_current_day_minus_daily_median_transaction_number_by_retrospective_time[
+ interval] = transaction_number_on_current_day - median_transaction_number
+
+ return [transaction_number_on_current_day_per_daily_average_transaction_number_by_retrospective_time,
+ transaction_number_on_current_day_minus_daily_average_transaction_number_by_retrospective_time,
+ transaction_number_on_current_day_per_daily_median_transaction_number_by_retrospective_time,
+ transaction_number_on_current_day_minus_daily_median_transaction_number_by_retrospective_time]
+
+ def get_generated_features_based_on_transaction_interval(self, current_timestamp,
+ time_stamps):
+ current_distance_between_transactions = current_timestamp - time_stamps[0] if time_stamps else 0
+
+ distances_by_retrospective_time = dict()
+ length_of_timestamp_collection = len(time_stamps)
+ for interval in self.intervals:
+ distances = list()
+ for i in range(0, length_of_timestamp_collection - 1, 1):
+ next_time_stamp = time_stamps[i + 1]
+ current_time_stamp = time_stamps[i]
+ distance_between_transactions = current_time_stamp - next_time_stamp
+ distances.append(distance_between_transactions)
+ distances_by_retrospective_time[interval] = distances
+
+ average_distance_between_transactions_by_retrospective_time = dict()
+ median_distance_between_transactions_by_retrospective_time = dict()
+ for interval in self.intervals:
+ distances = distances_by_retrospective_time.get(interval)
+ average_distance_between_transactions_by_retrospective_time[interval] = statistics.mean(
+ distances) if distances else 0
+ median_distance_between_transactions_by_retrospective_time[interval] = statistics.median(
+ distances) if distances else 0
+
+ average_distance_per_current_distance_between_transactions_by_retrospective_time = dict()
+ average_distance_time_minus_current_distance_between_transactions_by_retrospective_time = dict()
+ median_distance_per_current_distance_between_transactions_by_retrospective_time = dict()
+ median_distance_minus_current_distance_between_transactions_by_retrospective_time = dict()
+
+ for interval in self.intervals:
+ average_distance = average_distance_between_transactions_by_retrospective_time.get(interval)
+ median_distance = median_distance_between_transactions_by_retrospective_time.get(interval)
+ average_distance_per_current_distance_between_transactions_by_retrospective_time[
+ interval] = average_distance / current_distance_between_transactions if current_distance_between_transactions != 0 else 0
+ average_distance_time_minus_current_distance_between_transactions_by_retrospective_time[
+ interval] = average_distance - current_distance_between_transactions
+ median_distance_per_current_distance_between_transactions_by_retrospective_time[
+ interval] = median_distance / current_distance_between_transactions if current_distance_between_transactions != 0 else 0
+ median_distance_minus_current_distance_between_transactions_by_retrospective_time[
+ interval] = median_distance - current_distance_between_transactions
+
+ return [average_distance_per_current_distance_between_transactions_by_retrospective_time,
+ average_distance_time_minus_current_distance_between_transactions_by_retrospective_time,
+ median_distance_per_current_distance_between_transactions_by_retrospective_time,
+ median_distance_minus_current_distance_between_transactions_by_retrospective_time]
--- /dev/null
+from enum import Enum, unique
+
+
+@unique
+class Error(Enum):
+ ddl_build_error = 1
+ fit_launch_condition_error = 2
--- /dev/null
+import error_module
+
+
+class CommonCustomException(Exception):
+ def __init__(self, error_message):
+ self.error_message = error_message
+
+
+class DdlBuildException(CommonCustomException):
+ def __init__(self, error_message):
+ CommonCustomException.__init__(self, error_message)
+ self.error = error_module.Error.ddl_build_error
+
+class FitLaunchConditionException(CommonCustomException):
+ def __init__(self, error_message):
+ CommonCustomException.__init__(self, error_message)
+ self.error = error_module.Error.fit_launch_condition_error
\ No newline at end of file
--- /dev/null
+click==8.0.3
+colorama==0.4.4
+Flask==2.0.2
+imbalanced-learn==0.9.0
+imblearn==0.0
+importlib-metadata==4.10.1
+itsdangerous==2.0.1
+Jinja2==3.0.3
+joblib==1.1.0
+lightgbm==3.3.2
+MarkupSafe==2.0.1
+mysql-connector==2.2.9
+mysql-connector-python==8.0.28
+numpy==1.21.5
+pandas==1.3.5
+pika==1.2.0
+protobuf==3.19.3
+python-dateutil==2.8.2
+pytz==2021.3
+scikit-learn==1.0.2
+scipy==1.7.3
+six==1.16.0
+threadpoolctl==3.0.0
+typing-extensions==4.0.1
+Werkzeug==2.0.2
+xgboost==1.5.2
+zipp==3.7.0
--- /dev/null
+import csv
+import json
+import logging
+import random
+
+import math
+import os
+import time
+from threading import Thread
+
+import pika
+from flask import Flask, jsonify, request, Response
+from imblearn.combine import SMOTEENN
+from imblearn.over_sampling import RandomOverSampler
+from imblearn.under_sampling import RandomUnderSampler
+from lightgbm import LGBMClassifier
+from sklearn.decomposition import PCA, TruncatedSVD
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.feature_selection import RFECV
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import confusion_matrix, accuracy_score, balanced_accuracy_score, recall_score, f1_score, \
+ roc_auc_score
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import GaussianNB
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler, QuantileTransformer, Normalizer, RobustScaler, MaxAbsScaler, \
+ MinMaxScaler
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.utils.validation import column_or_1d
+from xgboost import XGBClassifier
+
+import database_module
+import exception_module
+import error_module
+
+DEFAULT_INSERT_BATCH_SIZE = 1000
+PREVIOUSLY_DONE = "PREVIOUSLY_DONE"
+STARTED = "STARTED"
+FINISHED = "FINISHED"
+ENCODING = "ENCODING"
+FEATURE_ENGINEERING = "FEATURE_ENGINEERING"
+SAMPLING = "SAMPLING"
+FITTING = "FITTING"
+ESTIMATOR_PERSISTING = "ESTIMATOR_PERSISTING"
+METRIC_PERSISTING = "METRIC_PERSISTING"
+IN_PROGRESS = "IN_PROGRESS"
+DONE = "DONE"
+FRAUD_TYPE = "fraud_type"
+
+app = Flask(__name__)
+
+
+class IllegalArgumentException(Exception):
+ pass
+
+
+class NoTimeBaseFieldInTableException(Exception):
+ pass
+
+
+class FieldNotApplicableForTimeBaseException(Exception):
+ pass
+
+
+class NoFraudTypeFieldException(Exception):
+ pass
+
+
+class TooManyFraudTypeFieldException(Exception):
+ pass
+
+
+class NoEnoughParameterException(Exception):
+ pass
+
+
+class FieldNotExistInTableException(Exception):
+ pass
+
+
+class EncodingNotApplicableException(Exception):
+ pass
+
+
+class FeatureEngineeringNotApplicableException(Exception):
+ pass
+
+
+class FieldNotApplicableForFraudException(Exception):
+ pass
+
+
+def send_async_message(id, method, status):
+ response = {
+ "train_task_id": id,
+ "method": method,
+ "status": status
+ }
+ try:
+ connection = pika.BlockingConnection(
+ pika.ConnectionParameters(host=rabbitmq_url, port=5672))
+ channel = connection.channel()
+ channel.queue_declare(queue='train', durable=True)
+ channel.basic_publish(exchange='', routing_key='train', body=json.dumps(response))
+ connection.close()
+ except (pika.exceptions.AMQPConnectionError, pika.exceptions.AMQPChannelError) as error:
+ logger.error(f"RabbitMQ error: {error}")
+
+
+def get_feature_selectors(cpu_core_num=1):
+ feature_selectors = {
+ 1: ('RFE', RFECV(estimator=RandomForestClassifier(n_jobs=cpu_core_num), n_jobs=cpu_core_num)),
+ 2: ('PCA', PCA(n_components=0.95, svd_solver='full')),
+ 3: ('SVD', TruncatedSVD())
+ }
+ return feature_selectors
+
+
+def get_samplers(cpu_core_num=1):
+ samplers = {
+ 1: ('RandomUnderSampler', RandomUnderSampler(sampling_strategy=0.5)),
+ 2: ('RandomOverSampler', RandomOverSampler(sampling_strategy=0.5)),
+ 3: ('SMOTEENN', SMOTEENN(sampling_strategy=0.5, n_jobs=cpu_core_num))
+ }
+ return samplers
+
+
+def get_scalers():
+ scalers = {
+ 1: ('StandardScaler', StandardScaler()),
+ 2: ('MinMaxScaler', MinMaxScaler()),
+ 3: ('MaxAbsScaler', MaxAbsScaler()),
+ 4: ('RobustScaler', RobustScaler()),
+ 5: ('QuantileTransformer-Normal', QuantileTransformer(output_distribution='normal')),
+ 6: ('QuantileTransformer-Uniform', QuantileTransformer(output_distribution='uniform')),
+ 7: ('Normalizer', Normalizer()),
+ }
+ return scalers
+
+
+def get_models(cpu_core_num=1):
+ models = {
+ 1: ('Logistic Regression', LogisticRegression(n_jobs=cpu_core_num)),
+ 2: ('LinearDiscriminantAnalysis', LinearDiscriminantAnalysis()),
+ 3: ('K-Nearest Neighbor', KNeighborsClassifier(n_jobs=cpu_core_num)),
+ 4: ('DecisionTree', DecisionTreeClassifier()),
+ 5: ('GaussianNB', GaussianNB()),
+ # 'SupportVectorMachine GPU': SupportVectorMachine(use_gpu=True),
+ # 'Random Forest GPU': RandomForestClassifier(use_gpu=True, gpu_ids=[0, 1], use_histograms=True),
+ 6: ('Random Forest', RandomForestClassifier(n_jobs=cpu_core_num)),
+ # 'MLP': MLPClassifier(),
+ 7: ('Light GBM', LGBMClassifier(n_jobs=cpu_core_num)),
+ # 'XGBoost': XGBClassifier(tree_method='gpu_hist', gpu_id=0)
+ 8: ('XGBoost', XGBClassifier())
+ }
+ return models
+
+
+@app.route('/liveness', methods=['GET'])
+def get_liveness():
+ result = {
+ "response": "OK"
+ }
+ return jsonify(result)
+
+
+@app.route('/cpu', methods=['GET'])
+def get_available_cpu_core():
+ core_number = os.cpu_count()
+ logger.debug(f"available cpu core number: {core_number}")
+ return jsonify(core_number)
+
+
+@app.route('/upload', methods=['POST'])
+def upload_train_database_from_csv():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ if request_parameter is None or not request_parameter:
+ error_message = "train database upload request's POST body is empty"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ try:
+ database_handler.upload_train_database(request_parameter)
+ except exception_module.DdlBuildException as e:
+ return create_error_response(e)
+ # details = e.args[0]
+ # error_message = details.get("error_message")
+ # error = e.error
+ # error_obj = {
+ # "error_name": e.error.name,
+ # "error_code": e.error.value,
+ # "error_message": e.error_message
+ # }
+ # return jsonify(error_obj),400
+ except database_module.IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ parameter_name = details.get("parameter_name")
+ value = details.get("value")
+ id = details.get("id")
+ error_message = f"{message}" + (
+ f", parameter name: {parameter_name}" if parameter_name is not None else "") + (
+ f", value: {value}" if value is not None else "") + (
+ f", id: {id}" if id is not None else "")
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ message = "Train database uploaded"
+ return Response(message, status=201)
+
+
+def create_error_response(common_exception):
+ error_obj = {
+ "error_name": common_exception.error.name,
+ "error_code": common_exception.error.value,
+ "error_message": common_exception.error_message
+ }
+ return jsonify(error_obj), 400
+
+
+@app.route('/schema', methods=['GET'])
+def get_schemas():
+ schemas = database_handler.get_schema_names()
+ return jsonify(schemas)
+
+
+@app.route('/table', methods=['GET'])
+def get_tables():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ tables = database_handler.get_table_names_of_given_database(schema_name)
+ return jsonify(tables)
+
+
+@app.route('/field_name_in_ordinal_position', methods=['GET'])
+def get_field_name_in_ordinal_position():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "Schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ field_name_in_ordinal_position = database_handler.get_field_name_in_ordinal_position(schema_name, table_name)
+ return jsonify(field_name_in_ordinal_position)
+
+
+@app.route('/field_type_in_ordinal_position', methods=['GET'])
+def get_field_type_in_ordinal_position():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "Schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ field_name_in_ordinal_position = database_handler.get_field_type_in_ordinal_position(schema_name, table_name)
+ return jsonify(field_name_in_ordinal_position)
+
+
+@app.route('/field_name_and_type', methods=['GET'])
+def get_field_type():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "Schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ field_types_by_name = database_handler.get_field_types_by_name(schema_name, table_name)
+ return jsonify(field_types_by_name)
+
+
+@app.route('/record', methods=['GET'])
+def get_records_with_limit_and_offset():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "Schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ limit = request.args.get("limit")
+ if limit is not None and type(limit) != int:
+ try:
+ limit = int(limit)
+ except ValueError:
+ message = "limit isn't integer"
+ logger.error(message)
+ return Response(message, 400)
+ offset = request.args.get("offset")
+ if offset is not None and type(offset) != int:
+ try:
+ offset = int(offset)
+ except ValueError:
+ message = "offset isn't integer"
+ logger.error(message)
+ return Response(message, 400)
+ result = database_handler.get_records_with_limit_and_offset(schema_name, table_name, limit, offset)
+ return jsonify(result)
+
+
+@app.route('/table_info', methods=['GET'])
+def get_table_info():
+ schema_name = request.args.get("schema_name")
+ table_name = request.args.get("table_name")
+ try:
+ check_schema(schema_name)
+ check_table(schema_name, table_name)
+ except IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message") + (
+ f", schema name: {schema_name}" if schema_name is not None and schema_name != "" else "") + (
+ f", table name: {table_name}" if table_name is not None and table_name != "" else "")
+ return Response(message, 400)
+ detailed_information_about_table = database_handler.get_detailed_information_about_table(schema_name=schema_name,
+ table_name=table_name)
+ return jsonify(detailed_information_about_table)
+
+
+def check_schema(schema_name):
+ if schema_name is None or schema_name == "":
+ message = "schema name is missing"
+ logger.error(message)
+ raise IllegalArgumentException(({"message": message}))
+ schemas = database_handler.get_schema_names()
+ if schema_name not in schemas:
+ message = "schema doesn't exist"
+ logger.error(message)
+ raise IllegalArgumentException({"message": message})
+
+
+def check_table(schema_name, table_name):
+ tables = database_handler.get_table_names_of_given_database(schema_name)
+ if table_name is None or table_name == "":
+ message = "table name is missing"
+ logger.error(message)
+ raise IllegalArgumentException({"message": message})
+ if table_name not in tables:
+ message = "table doesn't exist"
+ logger.error(message)
+ raise IllegalArgumentException({"message": message})
+
+
+@app.route('/encoding_and_feature_engineering_plan', methods=['POST'])
+def save_encoding_plan():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ if request_parameter is None or not request_parameter:
+ error_message = "encoding plan request's POST body is empty"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ schema_name = request_parameter.get("schema_name")
+ if schema_name is None:
+ error_message = "encoding plan request's schema name is missing"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ schemas = database_handler.get_schema_names()
+ if schema_name not in schemas:
+ error_message = "encoding plan request's schema name doesn't exist in database"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ table_name = request_parameter.get("table_name")
+ if table_name is None:
+ error_message = "encoding plan request's table name is missing"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ tables = database_handler.get_table_names_of_given_database(schema_name)
+ if table_name not in tables:
+ error_message = "encoding plan request's table name doesn't exist in the database"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ time_base_field_name = request_parameter.get("time_base_field_name")
+ if time_base_field_name is None or not time_base_field_name:
+ error_message = "encoding plan request's time base field name is missing"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ encoding_parameters = request_parameter.get("encoding_parameters")
+ if encoding_parameters is None or not encoding_parameters:
+ error_message = "encoding plan request's encoding parameters are missing"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ feature_engineering_parameters = request_parameter.get("feature_engineering_parameters")
+ if feature_engineering_parameters is None or not feature_engineering_parameters:
+ logger.info("feature engineering didn't set")
+
+ detailed_information_about_table = database_handler.get_detailed_information_about_table(schema_name=schema_name,
+ table_name=table_name)
+
+ try:
+ check_time_base_field_name(detailed_information_about_table, time_base_field_name)
+ check_if_plan_exist_yet(schema_name, table_name, time_base_field_name, encoding_parameters,
+ feature_engineering_parameters)
+ check_fields_and_possible_encoding_and_engineering(detailed_information_about_table,
+ encoding_parameters, feature_engineering_parameters)
+
+ except NoTimeBaseFieldInTableException as e:
+ details = e.args[0]
+ message = details.get("message")
+ logger.error(message)
+ return Response(message, status=400)
+ except FieldNotApplicableForTimeBaseException as e:
+ details = e.args[0]
+ message = details.get("message")
+ field_name = details.get("field_name")
+ error_message = f"{message}, field name: {field_name}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ id = details.get("id")
+ info_message = f"{message}, id: {id}"
+ logger.info(info_message)
+ return jsonify(id)
+ except NoEnoughParameterException as e:
+ details = e.args[0]
+ message = details.get("message")
+ parameter_name = details.get("parameter_name")
+ error_message = f"{message}, parameter: {parameter_name}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except FieldNotExistInTableException as e:
+ details = e.args[0]
+ message = details.get("message")
+ field_name = details.get("field_name")
+ error_message = f"{message}, schema name: {schema_name}, table name: {table_name}, field name: {field_name}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except FieldNotApplicableForFraudException as e:
+ details = e.args[0]
+ message = details.get("message")
+ field_name = details.get("field_name")
+ error_message = f"{message}, schema name: {schema_name}, table name: {table_name}, field name: {field_name}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except EncodingNotApplicableException as e:
+ details = e.args[0]
+ message = details.get("message")
+ field_name = details.get("field_name")
+ planned_encoding_type = details.get("planned_encoding_type")
+ error_message = f"{message}, schema name: {schema_name}, table name: {table_name}, field name: {field_name}, planned encoding type: {planned_encoding_type}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except NoFraudTypeFieldException as e:
+ error_message = "There aren't any fraud type field in encoding parameters"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except TooManyFraudTypeFieldException as e:
+ error_message = "There are too many fraud type field in encoding parameters"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except FeatureEngineeringNotApplicableException as e:
+ details = e.args[0]
+ message = details.get("message")
+ field_name = details.get("field_name")
+ field_type = details.get("field_type")
+ error_message = f"{message}" + (f", field name: {field_name}" if field_type is not None else "") + (
+ f", field type: {field_type}" if field_type is not None else "")
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ last_row_id = database_handler.persist_encoding_and_engineering_plan(schema_name, table_name,
+ detailed_information_about_table,
+ time_base_field_name,
+ encoding_parameters,
+ feature_engineering_parameters)
+ return jsonify(last_row_id)
+
+
+@app.route('/encoding_and_feature_engineering_plan', methods=['GET'])
+def get_encoded_table():
+ id = request.args.get("id")
+ try:
+ if id is None or id == "":
+ result = database_handler.get_all_encoding_and_feature_engineering_plan()
+ else:
+ result = database_handler.get_encoding_and_feature_engineering_plan_by_id(id)
+ except database_module.NoneException as e:
+ details = e.args[0]
+ message = details.get("message")
+ parameter = details.get("parameter")
+ error_message = f"{message}, id: {parameter}"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ return jsonify(result)
+
+
+@app.route('/available_feature_selector', methods=['GET'])
+def get_available_feature_selector():
+ result = dict()
+ feature_selectors = get_feature_selectors()
+ for key, value in feature_selectors.items():
+ result[key] = value[0]
+ return jsonify(result)
+
+
+@app.route('/available_sampler', methods=['GET'])
+def get_available_sampler():
+ result = dict()
+ samplers = get_samplers()
+ for key, value in samplers.items():
+ result[key] = value[0]
+ return jsonify(result)
+
+
+@app.route('/available_scaler', methods=['GET'])
+def get_available_scaler():
+ result = dict()
+ scalers = get_scalers()
+ for key, value in scalers.items():
+ result[key] = value[0]
+ return jsonify(result)
+
+
+@app.route('/available_model', methods=['GET'])
+def get_available_model():
+ result = dict()
+ models = get_models()
+ for key, value in models.items():
+ result[key] = value[0]
+ return jsonify(result)
+
+
+@app.route('/train_task', methods=['POST'])
+def save_train_task():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ feature_selector_keys = get_feature_selectors().keys()
+ sampler_keys = get_samplers().keys()
+ scaler_keys = get_scalers().keys()
+ model_keys = get_models().keys()
+ request_parameter = request.get_json()
+ if request_parameter is None:
+ error_message = "Train task POST body is empty"
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ try:
+ result = database_handler.persist_train_task(request_parameter, feature_selector_keys, sampler_keys,
+ scaler_keys, model_keys)
+ except database_module.IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ parameter_name = details.get("parameter_name")
+ value = details.get("value")
+ id = details.get("id")
+ error_message = f"{message}" + (
+ f", parameter name: {parameter_name}" if parameter_name is not None else "") + (
+ f", value: {value}" if value is not None else "") + (
+ f", id: {id}" if id is not None else "")
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ except database_module.TrainTaskExistYetException as e:
+ details = e.args[0]
+ message = details.get("message")
+ id = details.get("id")
+ info_message = f"{message}, id: {id}"
+ logger.info(info_message)
+ return jsonify(id)
+ return jsonify(result)
+
+
+@app.route('/train_task', methods=['GET'])
+def get_train_task():
+ id = request.args.get("id")
+ if id is None:
+ result = database_handler.get_all_train_task()
+ else:
+ result = database_handler.get_train_task_by_id(id)
+ return jsonify(result)
+
+
+@app.route('/fit', methods=['POST'])
+def fit():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+
+ request_parameter = request.get_json()
+ train_task_id = request_parameter.get("train_task_id")
+ used_cpu_core = request_parameter.get("used_cpu_core")
+ available_cpu_core = os.cpu_count()
+ if used_cpu_core is None:
+ message = f"used cpu core number not set, use the whole available logical cpu core, core number: {available_cpu_core}"
+ logger.info(message)
+ used_cpu_core = available_cpu_core
+ else:
+ if used_cpu_core > available_cpu_core or used_cpu_core < 1:
+ message = f"To many or few cpu core has been set, use the whole available logical cpu core, core number: {available_cpu_core}"
+ logger.info(message)
+ used_cpu_core = available_cpu_core
+
+ train_task_status = database_handler.get_train_task_status(train_task_id)
+ try:
+ if train_task_status == IN_PROGRESS:
+ error_message = f"this train task is in progress yet, it can not be launched once more, train_task_id: {train_task_id}"
+ logger.error(error_message)
+ raise exception_module.FitLaunchConditionException(error_message)
+ if train_task_status == DONE:
+ error_message = f"this train task has been finished yet, train_task_id: {train_task_id}"
+ logger.error(error_message)
+ raise exception_module.FitLaunchConditionException(error_message)
+ is_any_train_task_in_progress = database_handler.check_if_any_train_in_progress()
+ if is_any_train_task_in_progress:
+ error_message = "train in progress yet, it can not be launched another train task"
+ logger.error(error_message)
+ raise exception_module.FitLaunchConditionException(error_message)
+ except exception_module.FitLaunchConditionException as e:
+ return create_error_response(e)
+
+ database_handler.set_train_task_status(train_task_id, IN_PROGRESS)
+
+ available_feature_selectors = get_feature_selectors(used_cpu_core)
+ available_samplers = get_samplers(used_cpu_core)
+ available_scalers = get_scalers()
+ available_models = get_models(used_cpu_core)
+ try:
+ if train_task_id is None:
+ message = "train_task_id not set"
+ raise IllegalArgumentException({"message": message})
+
+ train_parameters = database_handler.get_train_parameters(train_task_id)
+ if train_parameters is None:
+ message = "No train task belongs to this id"
+ logger.error(message)
+ raise IllegalArgumentException({"message": message})
+ transaction_identifier_field_name = train_parameters[1]
+ if transaction_identifier_field_name is None:
+ message = "Transaction identifier field name not set"
+ raise IllegalArgumentException({"message": message})
+
+ planned_encoding_and_feature_engineering_id = train_parameters[3]
+ if planned_encoding_and_feature_engineering_id is None:
+ message = "Planned encoding and feature engineering id not set"
+ raise IllegalArgumentException({"message": message})
+
+ feature_selector_key = train_parameters[4]
+ if feature_selector_key is None:
+ message = "Feature selector not set"
+ raise IllegalArgumentException({"message": message})
+
+ feature_selector_name = available_feature_selectors.get(int(feature_selector_key))[0]
+ feature_selector = available_feature_selectors.get(int(feature_selector_key))[1]
+ logger.info(f"Feature selector: {feature_selector_name}")
+ if feature_selector is None:
+ message = "Feature selector doesn't exist in train application"
+ raise IllegalArgumentException({"message": message})
+
+ sampler_key = train_parameters[5]
+ if sampler_key is None:
+ message = "Sampler not set"
+ raise IllegalArgumentException({"message": message})
+ sampler_name = available_samplers.get(int(sampler_key))[0]
+ sampler = available_samplers.get(int(sampler_key))[1]
+ logger.info(f"Sampler: {sampler_name}")
+ if sampler is None:
+ message = "Sampler doesn't exist in train application"
+ raise IllegalArgumentException({"message": message})
+
+ scaler_key = train_parameters[6]
+ if scaler_key is None:
+ message = "Scaler not set"
+ raise IllegalArgumentException({"message": message})
+ scaler_name = available_scalers.get(int(scaler_key))[0]
+ scaler = available_scalers.get(int(scaler_key))[1]
+ logger.info(f"Scaler: {scaler_name}")
+ if scaler is None:
+ message = "Scaler doesn't exist in train application"
+ raise IllegalArgumentException({"message": message})
+
+ model_key = train_parameters[7]
+ if model_key is None:
+ message = "Model not set"
+ raise IllegalArgumentException({"message": message})
+ model_name = available_models.get(int(model_key))[0]
+ model = available_models.get(int(model_key))[1]
+ logger.info(f"Model: {model_name}")
+ if model is None:
+ message = "Model doesn't exist in train application"
+ raise IllegalArgumentException({"message": message})
+
+ split_test_size = None
+ saved_test_split_size = train_parameters[8]
+ if saved_test_split_size is None:
+ logger.info("No test split size set, use default value, 0.25")
+ else:
+ split_test_size = saved_test_split_size
+ logger.info(f"Train test split size: {split_test_size}")
+
+ fit_process_parameter = dict()
+ fit_process_parameter["transaction_identifier_field_name"] = transaction_identifier_field_name
+ fit_process_parameter["train_task_id"] = train_task_id
+ fit_process_parameter[
+ "planned_encoding_and_feature_engineering_id"] = planned_encoding_and_feature_engineering_id
+ fit_process_parameter["feature_selector"] = feature_selector
+ fit_process_parameter["sampler"] = sampler
+ fit_process_parameter["scaler"] = scaler
+ fit_process_parameter["model"] = model
+ fit_process_parameter["used_cpu_core"] = used_cpu_core
+ fit_process_parameter["split_test_size"] = split_test_size
+ thread = Thread(target=process_fit, args=(fit_process_parameter,))
+ thread.start()
+ logger.info(f"Train parameters are proper, fit process started, train_task_id: {train_task_id}")
+ response = Response(status=200)
+ except IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ logger.error(message)
+ request_message = message + (
+ f", train_task_id: {train_task_id}" if train_task_id is not None else "")
+ response = Response(request_message, status=400)
+ return response
+
+
+@app.route('/metrics', methods=['GET'])
+def get_metrics():
+ estimator_id = request.args.get("id")
+ if estimator_id is None:
+ result = database_handler.get_all_metrics()
+ else:
+ result = database_handler.get_metrics_by_id(estimator_id)
+ return jsonify(result)
+
+
+@app.route('/estimator', methods=['GET'])
+def get_estimators():
+ train_task_id = request.args.get("train_task_id")
+ if train_task_id is None:
+ result = database_handler.get_all_estimator()
+ else:
+ result = database_handler.get_estimator_by_train_task_id(train_task_id)
+ return jsonify(result)
+
+
+@app.route('/transaction_identifier_field_name', methods=['GET'])
+def get_transaction_identifier_field_name():
+ id = request.args.get("id")
+ if id is None:
+ message = "id is missing"
+ logger.error(message)
+ return Response(message, status=400)
+ field_name = database_handler.get_transaction_identifier_field_name_by_estimator_id(id)
+ return field_name
+
+
+@app.route('/confusion_matrix', methods=['GET'])
+def get_confusion_matrix():
+ train_task_id = request.args.get("train_task_id")
+ if train_task_id is None:
+ message = "Train task id is missing"
+ logger.error(message)
+ return Response(message, status=400)
+ else:
+ result = database_handler.get_confusion_matrix_elements_by_train_task_id(train_task_id)
+ return jsonify(result)
+
+
+@app.route('/train_task_csv', methods=['POST'])
+def create_train_task_csv():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ path = request_parameter.get("path")
+ file_name = request_parameter.get("file_name")
+ date_time_to_append = request_parameter.get("date_time_to_append")
+ available_feature_selectors = get_feature_selectors()
+ available_samplers = get_samplers()
+ available_scalers = get_scalers()
+ available_models = get_models()
+ try:
+ database_handler.create_train_task_csv_file(path, file_name, date_time_to_append, available_feature_selectors,
+ available_samplers,
+ available_scalers, available_models)
+ except IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ logger.error(message)
+ return Response(message, status=400)
+ return Response(status=200)
+
+
+@app.route('/train_task_csv_content', methods=['GET'])
+def get_train_task_csv_content():
+ available_feature_selectors = get_feature_selectors()
+ available_samplers = get_samplers()
+ available_scalers = get_scalers()
+ available_models = get_models()
+ header_elements, lines = database_handler.create_train_task_csv_content(available_feature_selectors,
+ available_samplers,
+ available_scalers, available_models)
+ response = {
+ "header_elements": header_elements,
+ "lines": lines
+ }
+ return jsonify(response)
+
+
+@app.route('/metircs_csv', methods=['POST'])
+def create_metrics_csv():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ path = request_parameter.get("path")
+ file_name = request_parameter.get("file_name")
+ date_time_to_append = request_parameter.get("date_time_to_append")
+ try:
+ database_handler.create_metrics_csv_file(path, file_name, date_time_to_append)
+ except PermissionError as e:
+ message = f"Permission error occured writing csv file, path: {path}, file name: {file_name}"
+ logger.error(message)
+ return Response(message, status=400)
+ except csv.Error as e:
+ message = f"Csv error occured writing csv file, path: {path}, file name: {file_name}"
+ logger.error(message)
+ return Response(message, status=400)
+ except IOError as e:
+ message = f"IO error occured writing csv file, path: {path}, file name: {file_name}"
+ logger.error(message)
+ return Response(message, status=400)
+ return Response(status=200)
+
+
+@app.route('/metrics_csv_content', methods=['GET'])
+def get_metrics_csv_content():
+ header_elements, lines = database_handler.create_metrics_csv_content()
+ response = {
+ "header_elements": header_elements,
+ "lines": lines
+ }
+ return jsonify(response)
+
+
+@app.route('/train_task_name_by_estimator_id', methods=['GET'])
+def get_train_task_name_by_estimator_id():
+ estimator_id = request.args.get("estimator_id")
+ if estimator_id is None:
+ message = "Estimator id is missing"
+ logger.error(message)
+ return Response(message, status=400)
+ else:
+ result = database_handler.get_train_task_name_by_estimator_id(estimator_id)
+ if result is not None:
+ return result
+ else:
+ message = "Train task name didn't find by estimator id"
+ logger.error(message)
+ return Response(message, status=400)
+
+
+@app.route('/columns_contain_only_distinct_values', methods=['GET'])
+def get_columns_contain_only_distinct_values():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ response = database_handler.get_columns_contain_only_distinct_values(schema_name, table_name)
+ return jsonify(response)
+
+
+@app.route('/reduced_and_evaulation', methods=['POST'])
+def create_reduced_and_evaulated_tables():
+ if not request.is_json:
+ error_message = "request body must be JSON"
+ logger.error(error_message)
+ return Response(error_message, status=415)
+ request_parameter = request.get_json()
+ schema_name = request_parameter.get("schema_name")
+ table_name = request_parameter.get("table_name")
+ fraud_type_field_name = request_parameter.get("fraud_type_field_name")
+ positive_fraud_case_number = request_parameter.get("positive_fraud_case_number")
+ negative_fraud_case_number = request_parameter.get("negative_fraud_case_number")
+ date_suffix_to_append = request_parameter.get("date_suffix_to_append")
+ try:
+ csv_content_and_planned_file_name = database_handler.create_reduced_and_for_evaluation_tables(schema_name,
+ table_name,
+ fraud_type_field_name,
+ positive_fraud_case_number,
+ negative_fraud_case_number,
+ date_suffix_to_append)
+ except database_module.IllegalArgumentException as e:
+ details = e.args[0]
+ message = details.get("message")
+ parameter_name = details.get("parameter_name")
+ error_message = f"{message}" + (f", parameter name: {parameter_name}" if parameter_name is not None else "")
+ logger.error(error_message)
+ return Response(error_message, status=400)
+ return jsonify(csv_content_and_planned_file_name)
+
+
+@app.route('/generic_records', methods=['GET'])
+def get_generic_records_from_evaluation_table():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ response = database_handler.get_generic_records_for_evaluation(schema_name, table_name)
+ return jsonify(response)
+
+@app.route('/max_retrospective_day', methods=['GET'])
+def get_max_retrospective_day():
+ schema_name = request.args.get("schema_name")
+ if schema_name is None:
+ message = "schema name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ table_name = request.args.get("table_name")
+ if table_name is None:
+ message = "table name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ time_base_field_name= request.args.get("time_base_field_name")
+ if time_base_field_name is None:
+ message = "time_base_field_name is missing"
+ logger.error(message)
+ return Response(message, 400)
+ max_retrospective_day=database_handler.get_max_retrospective_day(schema_name,table_name,time_base_field_name)
+ return jsonify(max_retrospective_day)
+
+
+def check_time_base_field_name(detailed_information_about_table, time_base_field_name):
+ field_names = list()
+ time_base_field_names = list()
+ field_properties = detailed_information_about_table.get("fields")
+ for field_property in field_properties:
+ field_type = field_property.get("type")
+ field_name = field_property.get("name")
+ field_names.append(field_name)
+ if field_type == "datetime":
+ time_base_field_names.append(field_name)
+ if time_base_field_name not in field_names:
+ raise FieldNotExistInTableException(
+ {"message": "The field doesn't exist in this database", "field_name": time_base_field_name})
+ if not time_base_field_names:
+ raise NoTimeBaseFieldInTableException({"message": "There isn't any time base field in the table"})
+ if not time_base_field_name in time_base_field_names:
+ raise FieldNotApplicableForTimeBaseException(
+ {"message": "The specified field unsuitable for time base in the table",
+ "field_name": time_base_field_name})
+
+
+def check_if_plan_exist_yet(schema_name, table_name, time_base_field_name, encoding_parameters,
+ feature_engineering_parameters):
+ existing_plans = database_handler.get_all_encoding_and_feature_engineering_plan()
+ for plan in existing_plans:
+ if plan.get("schema_name") == schema_name and plan.get("table_name") == table_name and plan.get(
+ "encoding_parameters") == encoding_parameters and plan.get(
+ "feature_engineering_parameters") == feature_engineering_parameters and plan.get(
+ "time_base_field_name") == time_base_field_name:
+ message = "Encoding and feature engineering plan exist yet"
+ id = plan.get("id")
+ raise IllegalArgumentException({"message": message, "id": id})
+
+
+def check_fields_and_possible_encoding_and_engineering(detailed_information_about_table,
+ encoding_parameters, feature_engineering_parameters):
+ field_properties = detailed_information_about_table.get("fields")
+ number_of_int_and_float_type_fields = get_number_of_int_and_float_typed_fields(field_properties)
+ field_names_as_keys_in_encoding_parameters = encoding_parameters.keys()
+ number_of_encoding_parameters = len(field_names_as_keys_in_encoding_parameters)
+ number_of_fields_in_current_table = len(field_properties)
+ number_of_fields_to_be_encoded = number_of_fields_in_current_table - number_of_int_and_float_type_fields - 2 # fraud and id
+ if number_of_encoding_parameters < number_of_fields_to_be_encoded:
+ raise NoEnoughParameterException(
+ {"message": "there aren't enough parameter to encode", "parameter_name": encoding_parameters})
+ field_names_in_table = get_field_names_from_field_and_type_properties(field_properties)
+ for field in field_names_as_keys_in_encoding_parameters:
+ if field not in field_names_in_table:
+ raise FieldNotExistInTableException(
+ {"message": "The field doesn't exist in this database", "field_name": field})
+ if encoding_parameters.get(field) == "fraud_type":
+ check_if_field_applicable_for_fraud(detailed_information_about_table, field)
+
+ all_field_names = check_encoding_parameters(encoding_parameters,
+ detailed_information_about_table)
+ check_number_of_fraud_type_field(all_field_names, encoding_parameters)
+ if feature_engineering_parameters is not None:
+ check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+
+def check_encoding_parameters(encoding_parameters, detailed_information_about_table):
+ all_field_names = list()
+ field_properties = detailed_information_about_table.get("fields")
+ for field_property in field_properties:
+ field_name = field_property.get("name")
+ all_field_names.append(field_name)
+ if field_name in encoding_parameters.keys():
+ planned_encoding_type = encoding_parameters.get(field_name)
+ if (planned_encoding_type == "julian" and field_property.get(
+ "type") != "datetime") or (planned_encoding_type == "label_encoder" and field_property.get(
+ "type") != "varchar") or (planned_encoding_type == "int" and "int" not in field_property.get(
+ database_module.OTHER_APPLICABLE_ENCODING_TYPES)) or (
+ planned_encoding_type == "float" and "float" not in field_property.get(
+ database_module.OTHER_APPLICABLE_ENCODING_TYPES)) or planned_encoding_type not in ["julian",
+ "label_encoder",
+ "int",
+ "float",
+ "fraud_type",
+ "omit"]:
+ raise EncodingNotApplicableException(
+ {"message": "the planned encoding can not applicable to this field", "field_name": field_name,
+ "planned_encoding_type": planned_encoding_type})
+ return all_field_names
+
+
+def get_number_of_int_and_float_typed_fields(field_and_type_and_other_eligible_type_for_encoding_collection):
+ int_types = ("tinyint", "smallint", "mediumint", "int", "bigint")
+ float_types = ("float", "double")
+ number_of_int_and_float_typed_field = 0
+ for field_property in field_and_type_and_other_eligible_type_for_encoding_collection:
+ current_type = field_property.get("type")
+ if current_type in int_types or current_type in float_types:
+ number_of_int_and_float_typed_field += 1
+ return number_of_int_and_float_typed_field
+
+
+def check_if_field_applicable_for_fraud(detailed_information_about_table, current_field_name):
+ fraud_candidates = detailed_information_about_table.get("fraud_candidates")
+ applicable_field_names_for_fraud = list()
+ for fraud_candidate in fraud_candidates:
+ applicable_field_names_for_fraud.append(fraud_candidate.get("name"))
+ if current_field_name not in applicable_field_names_for_fraud:
+ raise FieldNotApplicableForFraudException(
+ {"message": "the field not applicable for fraud", "field_name": current_field_name})
+
+
+def check_number_of_fraud_type_field(field_names, encoding_parameters):
+ number_of_fraud_type_field = 0
+ for field_name in field_names:
+ if encoding_parameters.get(field_name) == "fraud_type":
+ number_of_fraud_type_field += 1
+ if number_of_fraud_type_field == 0:
+ raise NoFraudTypeFieldException()
+ if number_of_fraud_type_field > 1:
+ raise TooManyFraudTypeFieldException()
+
+
+def check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters):
+ field_name_to_be_engineered = feature_engineering_parameters.get("chosen_feature_field")
+ intervals = feature_engineering_parameters.get("intervals")
+ if field_name_to_be_engineered is None:
+ message = "field name for engineering is missing"
+ raise FeatureEngineeringNotApplicableException({"message": message})
+ if intervals is None or not intervals:
+ message = "interval list for engineering is missing or empty"
+ raise FeatureEngineeringNotApplicableException({"message": message})
+ field_properties = detailed_information_about_table.get("fields")
+ field_names = get_field_names_from_field_and_type_properties(field_properties)
+ if field_name_to_be_engineered not in field_names:
+ message = "the field name to be engineered doesn't exist in the table"
+ logger.error(message + f", field name to be engineered: {field_name_to_be_engineered}")
+ raise FeatureEngineeringNotApplicableException({"message": message})
+ for field_property in field_properties:
+ field_name = field_property.get("name")
+ field_type = field_property.get("type")
+ if field_name == field_name_to_be_engineered:
+ if field_type not in ["int", "float"]:
+ message = "only int or float type field can be used for feature engineering"
+ raise FeatureEngineeringNotApplicableException(
+ {"message": message, "field name": field_name, "field's type": field_type})
+ if encoding_parameters.get(field_name) == "omit":
+ message = f"the {field_name} field has been omitted during encoding process, it can't be used for feature engineering"
+ raise FeatureEngineeringNotApplicableException({"message": message})
+ if len(intervals) == 0:
+ message = "time intervals list is empty"
+ raise FeatureEngineeringNotApplicableException({"message": message})
+
+
+def get_field_names_from_field_and_type_properties(field_and_type_properties):
+ result = list()
+ for item in field_and_type_properties:
+ result.append(item.get("name"))
+ return result
+
+
+def process_fit(parameter):
+ message = dict()
+ transaction_identifier_field_name = parameter.get("transaction_identifier_field_name")
+ train_task_id = parameter.get("train_task_id")
+ logger.info(f"train process started, train_task_id: {train_task_id}")
+ planned_encoding_and_feature_engineering_id = parameter.get("planned_encoding_and_feature_engineering_id")
+ schema_name, table_name, detailed_information_about_table, time_base_field_name, encoding_parameters, feature_engineering_parameters = database_handler.get_parameters_from_plan_by_id(
+ planned_encoding_and_feature_engineering_id)
+ feature_selector = parameter.get("feature_selector")
+ sampler = parameter.get("sampler")
+ scaler = parameter.get("scaler")
+ model = parameter.get("model")
+ split_test_size = parameter.get("split_test_size")
+ used_cpu_core = parameter.get("used_cpu_core")
+
+ raw_dataset_id = database_handler.get_raw_dataset_id(schema_name, table_name)
+ encoding_id = database_handler.get_encoding_id_by_json_content(encoding_parameters)
+
+ encoded_table_registry_id, encoded_table_name, encoded_field_names, label_encoder_registry = database_handler.get_encoded_table_properties(
+ raw_dataset_id, encoding_id, time_base_field_name)
+ if encoded_table_name is not None:
+ logger.info(f"database encoded yet, train task id: {train_task_id}")
+ send_async_message(train_task_id, ENCODING, PREVIOUSLY_DONE)
+ dataset_to_train = database_handler.get_records_for_processing(schema_name, encoded_table_name,
+ time_base_field_name)
+ else:
+ start_encoding = time.time()
+ logger.info(f"database encoding started, train task id: {train_task_id}")
+ send_async_message(train_task_id, ENCODING, STARTED)
+ try:
+ dataset_to_train, encoded_table_registry_id, encoded_table_name, encoded_field_names, label_encoder_registry = database_handler.encode(
+ raw_dataset_id, schema_name, table_name, encoding_id, encoding_parameters, time_base_field_name)
+ except Exception as e:
+ logger.error("Error occured: " + str(e))
+
+ finish_encoding = time.time()
+ encoding_process_time = finish_encoding - start_encoding
+ logger.info(
+ f"database encoding finished, train task id: {train_task_id}, processing time: {encoding_process_time} sec")
+ send_async_message(train_task_id, ENCODING, FINISHED)
+
+ feature_engineered_insert_script_using_in_prediction = None
+ if feature_engineering_parameters is None:
+ logger.info("train without feature engineering")
+ else:
+ feature_engineering_id = database_handler.get_feature_engineering_id_by_json_content(
+ feature_engineering_parameters)
+ table_name, feature_engineered_insert_script = database_handler.get_table_name_from_feature_engineering_registry(
+ encoded_table_registry_id,
+ feature_engineering_id)
+ if table_name is not None:
+ logger.info(f"database feature engineered yet, train task id: {train_task_id}")
+ send_async_message(train_task_id, FEATURE_ENGINEERING, PREVIOUSLY_DONE)
+ dataset_to_train = database_handler.get_records_for_processing(schema_name, table_name,
+ time_base_field_name)
+ else:
+ start_feature_engineering = time.time()
+ logger.info(f"database feature engineering started, train task id: {train_task_id}")
+ send_async_message(train_task_id, FEATURE_ENGINEERING, STARTED)
+ fraud_type_field_name = get_fraud_type_field_name(encoding_parameters)
+ dataset_to_train, table_name, feature_engineered_insert_script_using_in_prediction = database_handler.feature_engineer(
+ schema_name, encoded_table_name, fraud_type_field_name,
+ feature_engineering_id,
+ dataset_to_train, time_base_field_name,
+ encoded_field_names,
+ feature_engineering_parameters,
+ encoded_table_registry_id, used_cpu_core)
+ finish_feature_engineering = time.time()
+ feature_engineering_process_time = finish_feature_engineering - start_feature_engineering
+ logger.info(
+ f"database feature engineering finished, train task id: {train_task_id}, processing time: {feature_engineering_process_time} sec")
+ send_async_message(train_task_id, FEATURE_ENGINEERING, FINISHED)
+
+ features = dataset_to_train[:, :-1]
+ labels = dataset_to_train[:, -1:]
+ labels = labels.astype(int)
+ labels = column_or_1d(labels)
+ train_features, test_features, train_labels, test_labels = train_test_split(features, labels,
+ test_size=split_test_size,
+ random_state=0)
+
+ logger.info("sampling started")
+ send_async_message(train_task_id, SAMPLING, STARTED)
+ start_sampling = time.time()
+ sampled_train_features, sampled_train_labels = sampler.fit_resample(train_features, train_labels)
+ finish_sampling = time.time()
+ sampling_proessing_time = finish_sampling - start_sampling
+ logger.info(
+ f"sampling finished, train task id: {train_task_id}, processing time: {sampling_proessing_time} sec")
+ send_async_message(train_task_id, SAMPLING, FINISHED)
+
+ pipeline = Pipeline(
+ [('scaler', scaler), ('featureSelector', feature_selector), ('model', model)]
+ )
+ logger.info(f"pipeline's training started, train task id: {train_task_id}")
+ send_async_message(train_task_id, FITTING, STARTED)
+ start_fit = time.time()
+ pipeline.fit(sampled_train_features, sampled_train_labels)
+ finish_fit = time.time()
+ fit_processing_time = finish_fit - start_fit
+ logger.info(
+ f"pipeline's training finished, train task id: {train_task_id} ,processing time: {fit_processing_time} sec")
+ send_async_message(train_task_id, FITTING, FINISHED)
+ predicted_labels = pipeline.predict(test_features)
+ conf_matrix = confusion_matrix(test_labels, predicted_labels)
+ logger.info(f"Confusion Matrix: {conf_matrix}, train_task_id: {train_task_id}")
+
+ estimator = dict()
+ estimator["pipeline"] = pipeline
+ estimator["label_encoder_registry"] = label_encoder_registry
+ estimator["encoding_parameters"] = encoding_parameters
+ estimator["feature_engineering_parameters"] = feature_engineering_parameters
+ estimator["encoded_field_names"] = encoded_field_names
+ estimator["time_base_field_name"] = time_base_field_name
+ estimator["schema_name"] = schema_name
+ estimator["feature_engineered_table_name"] = table_name
+ estimator["detailed_information_about_table"] = detailed_information_about_table
+ estimator["feature_engineered_insert_script"] = feature_engineered_insert_script_using_in_prediction
+
+ send_async_message(train_task_id, ESTIMATOR_PERSISTING, STARTED)
+ estimator_id = database_handler.persist_estimator(train_task_id, transaction_identifier_field_name, estimator)
+ logger.info(
+ f"estimator (pipeline, label encoders, encoding parameters, feature engineering parameters etc.) persisted, train task id: {train_task_id}, estimator_id: {estimator_id}")
+ send_async_message(train_task_id, ESTIMATOR_PERSISTING, FINISHED)
+
+ send_async_message(train_task_id, METRIC_PERSISTING, STARTED)
+ calculate_and_save_metrics(estimator_id, conf_matrix, test_labels, predicted_labels)
+ logger.info(f"Metrics persisted, train task id: {train_task_id}")
+ send_async_message(train_task_id, METRIC_PERSISTING, FINISHED)
+
+ database_handler.set_train_task_status(train_task_id, DONE)
+ logger.info(f"train process finished, train_task_id: {train_task_id}")
+
+
+def get_fraud_type_field_name(encoding_parameters):
+ for field_name, encoding_type in encoding_parameters.items():
+ if encoding_type == FRAUD_TYPE:
+ return field_name
+
+
+def calculate_and_save_metrics(estimator_id, conf_matrix, test_labels, predicted_labels):
+ TP = int(conf_matrix[0][0])
+ FP = int(conf_matrix[0][1])
+ FN = int(conf_matrix[1][0])
+ TN = int(conf_matrix[1][1])
+ temp = TP + FN
+ sensitivity = 0
+ if temp != 0:
+ sensitivity = TP / (TP + FN)
+ temp = TN + FP
+ specificity = 0
+ if temp != 0:
+ specificity = TN / (TN + FP)
+ accuracy = accuracy_score(test_labels, predicted_labels)
+ balanced_accuracy = balanced_accuracy_score(test_labels, predicted_labels)
+ precision = 0
+ temp = TP + FP
+ if temp != 0:
+ precision = TP / (TP + FP)
+ recall = recall_score(test_labels, predicted_labels)
+ temp = TP + FN
+ PPV = 0
+ if temp != 0:
+ PPV = TP / (TP + FN)
+ temp = TN + FN
+ NPV = 0
+ if temp != 0:
+ NPV = TN / (TN + FN)
+ temp = FN + TP
+ FNR = 0
+ if temp != 0:
+ FNR = FN / (FN + TP)
+ temp = FP + TN
+ FPR = 0
+ if temp != 0:
+ FPR = FP / (FP + TN)
+ FDR = 0
+ temp = FP + TP
+ if temp != 0:
+ FDR = FP / (FP + TP)
+ temp = FN + TN
+ FOR = 0
+ if temp != 0:
+ FOR = FN / (FN + TN)
+ f1 = f1_score(test_labels, predicted_labels)
+ f_05 = calculateF(0.5, precision, recall)
+ f2 = calculateF(2, precision, recall)
+ temp = math.sqrt(TP + FP) * math.sqrt(TP + FN) * math.sqrt(TN + FP) * math.sqrt(TN + FN)
+ MCC = 0
+ if temp != 0:
+ MCC = (TP * TN - FP * FN) / temp
+ ROCAUC = roc_auc_score(test_labels, predicted_labels)
+ Youdens_statistic = sensitivity + specificity - 1
+ database_handler.persist_metrics(estimator_id, TP, FP, TN, FN, sensitivity, specificity, accuracy,
+ balanced_accuracy,
+ precision, recall, PPV, NPV, FNR, FPR, FDR, FOR, f1, f_05, f2, MCC, ROCAUC,
+ Youdens_statistic)
+
+
+def calculateF(beta, precision, recall):
+ """F érték számítása"""
+ temp = beta * beta * precision + recall
+ if temp != 0:
+ f_beta = (1 + beta) * (1 + beta) * precision * recall / temp
+ else:
+ f_beta = 0
+ return f_beta
+
+
+if __name__ == "__main__":
+ log_file = os.getenv("TRAIN_LOG_FILE", "/home/tomi/log/train.log")
+ log_level = os.getenv("TRAIN_LOG_LEVEL", "DEBUG")
+ rabbitmq_url = os.getenv("RABBITMQ_URL", "localhost")
+ database_url = os.getenv("MYSQL_DATABASE_URL", "localhost")
+ database_user = os.getenv("MYSQL_USER", "root")
+ database_password = os.getenv("MYSQL_ROOT_PASSWORD", "pwd")
+ file_handler = logging.FileHandler(log_file)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ file_handler.setFormatter(formatter)
+ logger = logging.getLogger("train.server")
+ logger.setLevel(log_level)
+ logger.addHandler(file_handler)
+ logger.debug("RabbitMQ: " + rabbitmq_url)
+ logger.debug("MYSQL_DATABASE_URL: " + database_url)
+ logger.debug("MYSQL_USER :" + database_user)
+ logger.debug("MYSQL_PASSWORD: " + database_password)
+ logger.debug("Train application started")
+ try:
+ database_handler = database_module.Handler(database_url, database_user, database_password)
+ database_handler.create_common_fraud_schemas()
+ except Exception as e:
+ logger.error("Can not connect to database")
+ logger.debug("train modul started")
+ app.run(host='0.0.0.0', port=8085, debug=True)
--- /dev/null
+import unittest
+import ddl_build_module
+# from src.database_module import IllegalArgumentException
+import exception_module
+
+
+class MyTestCase(unittest.TestCase):
+ def test_create_insert_script(self):
+ script = "INSERT INTO card.encoded (time,amount,vendor) VALUES(%s,%s,%s)"
+ builder = ddl_build_module.DdlCommandBuilder()
+ fields = ["time", "amount", "vendor"]
+ built_script = builder.create_insert_into_encoded_or_feature_engineered_script("card", "encoded", fields)
+ self.assertEqual(built_script, script) # add assertion here
+
+ def test_create_encoded_table_script(self):
+ script = "CREATE TABLE IF NOT EXISTS card.encoded(id BIGINT NOT NULL AUTO_INCREMENT,time DOUBLE PRECISION,amount DOUBLE PRECISION,vendor DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ fields = ["time", "amount", "vendor"]
+ built_script = builder.build_create_encoded_or_engineered_table_script("card", "encoded", fields)
+ self.assertEqual(script, built_script)
+
+ def test_build_create_generic_train_database_script(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME(6),amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(255)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ built_script = builder.build_create_generic_train_database_script(schema_name, table_name, field_type_by_name,True)
+ self.assertEqual(script, built_script)
+
+ def test_build_create_generic_train_database_script_if_missing_begin_bracket_thorws_exception(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar255)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_missing_end_bracket_thorws_exception(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(255",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_missing_varchar_number(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar()",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_varchar_number_not_int_type(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(1.5)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_varchar_number_not_number_type(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(a)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
--- /dev/null
+import unittest
+
+import encoding_module
+
+encoding_parameters = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+ "fraud": "fraud_type"
+}
+
+detailed_information_about_table = {
+ "fields": [
+ {
+ "name": "id",
+ "other_eligible_type_for_encoding": [],
+ "type": "bigint"
+ },
+ {
+ "name": "card_number",
+ "other_eligible_type_for_encoding": [
+ "float"
+ ],
+ "type": "varchar"
+ },
+ {
+ "name": "transaction_type",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "timestamp",
+ "other_eligible_type_for_encoding": [],
+ "type": "datetime"
+ },
+ {
+ "name": "amount",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "currency_name",
+ "other_eligible_type_for_encoding": [],
+ "type": "varchar"
+ },
+ {
+ "name": "response_code",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "country_name",
+ "other_eligible_type_for_encoding": [],
+ "type": "varchar"
+ },
+ {
+ "name": "vendor_code",
+ "other_eligible_type_for_encoding": [
+ "float",
+ "int"
+ ],
+ "type": "varchar"
+ },
+ {
+ "name": "fraud",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ }
+ ],
+ "fraud_candidates": [
+ {
+ "fraud_number": 601,
+ "name": "fraud",
+ "no_fraud_number": 11314
+ }
+ ],
+ "primary_key": "id",
+ "record_number": 11915
+}
+
+
+class TestEncoding(unittest.TestCase):
+ def test_create_modified_fields(self):
+ original_fields = ["id", "card_number", "transaction_type", "timestamp", "amount", "currency_name",
+ "response_code", "country_name", "vendor_code", "fraud"]
+ database_encoder = encoding_module.DataBaseEncoder(None, original_fields)
+ modified_fields = database_encoder.build_encoded_field_names(encoding_parameters, "id")
+ self.assertEqual(8, len(modified_fields))
+
+ def test_get_field_type_before_encoding(self):
+ original_fields = ["id", "card_number", "transaction_type", "timestamp", "amount", "currency_name",
+ "response_code", "country_name", "vendor_code", "fraud"]
+ database_encoder = encoding_module.DataBaseEncoder(None, original_fields)
+ field_type = database_encoder.get_field_type_before_encoding("amount", detailed_information_about_table)
+ self.assertEqual("int", field_type)
+
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+import server
+import unittest
+
+encoding_parametrs = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+}
+
+detailed_information_about_table = {
+ "fields": [
+ {
+ "name": "id",
+ "other_eligible_type_for_encoding": [],
+ "type": "bigint"
+ },
+ {
+ "name": "card_number",
+ "other_eligible_type_for_encoding": [
+ "float"
+ ],
+ "type": "varchar"
+ },
+ {
+ "name": "transaction_type",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "timestamp",
+ "other_eligible_type_for_encoding": [],
+ "type": "datetime"
+ },
+ {
+ "name": "amount",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "currency_name",
+ "other_eligible_type_for_encoding": [],
+ "type": "varchar"
+ },
+ {
+ "name": "response_code",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ },
+ {
+ "name": "country_name",
+ "other_eligible_type_for_encoding": [],
+ "type": "varchar"
+ },
+ {
+ "name": "vendor_code",
+ "other_eligible_type_for_encoding": [
+ "float",
+ "int"
+ ],
+ "type": "varchar"
+ },
+ {
+ "name": "fraud",
+ "other_eligible_type_for_encoding": [],
+ "type": "int"
+ }
+ ],
+ "fraud_candidates": [
+ {
+ "fraud_number": 601,
+ "name": "fraud",
+ "no_fraud_number": 11314
+ }
+ ],
+ "primary_key": [
+ "id"
+ ],
+ "record_number": 11915
+}
+
+
+def get_fields_from_detailed_information(detailed_information_about_table):
+ result = list()
+ complex_fields = detailed_information_about_table.get("fields")
+ for complex_field_item in complex_fields:
+ field_name = complex_field_item.get("name")
+ result.append(field_name)
+ return result
+
+
+class TestServer(unittest.TestCase):
+ def test_get_number_of_int_and_float_typed_fields(self):
+ number_of_numerical_fields = server.get_number_of_int_and_float_typed_fields(
+ detailed_information_about_table.get("fields"))
+ self.assertEqual(number_of_numerical_fields, 5)
+
+ def test_to_many_fraud_type_in_parameters_then_throws_exception(self):
+ encoding_parameters = {
+ "fraud": "fraud_type",
+ "transaction_type": "fraud_type"
+ }
+ with self.assertRaises(server.TooManyFraudTypeFieldException):
+ fields = get_fields_from_detailed_information(detailed_information_about_table)
+ server.check_number_of_fraud_type_field(fields, encoding_parameters)
+
+ def test_if_no_fraud_type_in_parameters_then_then_throws_exception(self):
+ encoding_parameters = {}
+ fields = get_fields_from_detailed_information(detailed_information_about_table)
+ with self.assertRaises(server.NoFraudTypeFieldException):
+ server.check_number_of_fraud_type_field(fields, encoding_parameters)
+
+ def test_if_use_unknown_encoding_type_then_check_encoding_parameters_throws_exception(self):
+ encoding_parameters = {
+ "vendor_code": "double"
+ }
+ with self.assertRaises(server.EncodingNotApplicableException):
+ server.check_encoding_parameters(encoding_parameters, detailed_information_about_table)
+
+ def test_if_use_wrong_encoding_type_then_check_encoding_parameters_throws_exception(self):
+ encoding_parameters = {
+ "vendor_code": "julian"
+ }
+ with self.assertRaises(server.EncodingNotApplicableException):
+ server.check_encoding_parameters(encoding_parameters, detailed_information_about_table)
+
+ def test_if_feature_engineering_not_complete_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "id",
+ "time_base_field_name": "timestamp"
+ }
+ encoding_parameters = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+ }
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+ def test_if_use_not_numerical_type_for_engineering_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "id",
+ "intervals": [3, 7],
+ "time_base_field_name": "timestamp"
+ }
+ encoding_parameters = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+ }
+
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+ def test_if_use_not_datetime_type_field_for_engineering_time_base_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "amount",
+ "intervals": [3, 7],
+ "time_base_field_name": "amount"
+ }
+ encoding_parameters = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+ }
+
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parametrs,
+ feature_engineering_parameters)
+
+ def test_if_use_not_empty_interval_list_for_engineering_time_base_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "amount",
+ "intervals": [],
+ "time_base_field_name": "timestamp"
+ }
+ encoding_parameters = {
+ "card_number": "label_encoder",
+ "transaction_type": "omit",
+ }
+
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+ def test_if_omit_feature_engineering_field_during_encoding_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "amount",
+ "intervals": [2, 5],
+ "time_base_field_name": "timestamp"
+ }
+ encoding_parameters = {
+ "amount": "omit",
+ "transaction_type": "omit"
+ }
+
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+ def test_if_omit_feature_engineering_time_base_field_during_encoding_then_check_throws_exception(self):
+ feature_engineering_parameters = {
+ "feature_name_for_engineering": "amount",
+ "intervals": [2, 5],
+ "time_base_field_name": "timestamp"
+ }
+ encoding_parameters = {
+ "timestamp": "omit",
+ "transaction_type": "omit"
+ }
+
+ with self.assertRaises(server.FeatureEngineeringNotApplicableException):
+ server.check_if_feature_engineering_applicable(detailed_information_about_table, encoding_parameters,
+ feature_engineering_parameters)
+
+ def test_get_fraud_type_field_name(self):
+ encoding_parameters = {
+ "card_number": "float",
+ "timestamp": "julian",
+ "currency_name": "label_encoder",
+ "fraud": "fraud_type",
+ "country_name": "omit",
+ "vendor_code": "label_encoder"
+ }
+ fraud_type_field_name=server.get_fraud_type_field_name(encoding_parameters)
+ self.assertEqual(fraud_type_field_name, "fraud")
+
+
+if __name__ == "__main__":
+ unittest.main()
--- /dev/null
+import unittest
+import ddl_build_module
+from src.database_module import IllegalArgumentException
+import exception_module
+
+
+class MyTestCase(unittest.TestCase):
+ def test_create_insert_script(self):
+ script = "INSERT INTO card.encoded (time,amount,vendor) VALUES(%s,%s,%s)"
+ builder = ddl_build_module.DdlCommandBuilder()
+ fields = ["time", "amount", "vendor"]
+ built_script = builder.create_insert_into_encoded_or_feature_engineered_script("card", "encoded", fields)
+ self.assertEqual(built_script, script) # add assertion here
+
+ def test_create_encoded_table_script(self):
+ script = "CREATE TABLE IF NOT EXISTS card.encoded(id BIGINT NOT NULL AUTO_INCREMENT,time DOUBLE PRECISION,amount DOUBLE PRECISION,vendor DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ fields = ["time", "amount", "vendor"]
+ built_script = builder.build_create_encoded_or_engineered_table_script("card", "encoded", fields)
+ self.assertEqual(script, built_script)
+
+ def test_build_create_generic_train_database_script(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME(6),amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(255)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ built_script = builder.build_create_generic_train_database_script(schema_name, table_name, field_type_by_name,True)
+ self.assertEqual(script, built_script)
+
+ def test_build_create_generic_train_database_script_if_missing_begin_bracket_thorws_exception(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar255)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_missing_end_bracket_thorws_exception(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(255",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_missing_varchar_number(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar()",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_varchar_number_not_int_type(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(1.5)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+ def test_build_create_generic_train_database_script_if_varchar_number_not_number_type(self):
+ script = "CREATE TABLE IF NOT EXISTS card.transaction(id BIGINT NOT NULL AUTO_INCREMENT,card_number VARCHAR(255),timestamp DATETIME,amount INT,val DOUBLE PRECISION,PRIMARY KEY (id)) ENGINE = InnoDB"
+ builder = ddl_build_module.DdlCommandBuilder()
+ schema_name="card"
+ table_name="transaction"
+ field_type_by_name={
+ "card_number":"varchar(a)",
+ "timestamp":"datetime",
+ "amount":"int",
+ "val":"double precision"
+ }
+ with self.assertRaises(exception_module.DdlBuildException):
+ builder.build_create_generic_train_database_script("card", "encoded", field_type_by_name,True)
+
+if __name__ == '__main__':
+ unittest.main()
--- /dev/null
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
--- /dev/null
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# M2_HOME - location of maven2's installed home dir
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+ if [ -f /usr/local/etc/mavenrc ] ; then
+ . /usr/local/etc/mavenrc
+ fi
+
+ if [ -f /etc/mavenrc ] ; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ] ; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ MINGW*) mingw=true;;
+ Darwin*) darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ export JAVA_HOME="`/usr/libexec/java_home`"
+ else
+ export JAVA_HOME="/Library/Java/Home"
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=`java-config --jre-home`
+ fi
+fi
+
+if [ -z "$M2_HOME" ] ; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+
+ saveddir=`pwd`
+
+ M2_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ M2_HOME=`cd "$M2_HOME" && pwd`
+
+ cd "$saveddir"
+ # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --unix "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME="`(cd "$M2_HOME"; pwd)`"
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="`which javac`"
+ if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=`which readlink`
+ if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
+ if $darwin ; then
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
+ else
+ javaExecutable="`readlink -f \"$javaExecutable\"`"
+ fi
+ javaHome="`dirname \"$javaExecutable\"`"
+ javaHome=`expr "$javaHome" : '\(.*\)/bin'`
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="`\\unset -f command; \\command -v java`"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+ if [ -z "$1" ]
+ then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ] ; do
+ if [ -d "$wdir"/.mvn ] ; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=`cd "$wdir/.."; pwd`
+ fi
+ # end of workaround
+ done
+ echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ echo "$(tr -s '\n' ' ' < "$1")"
+ fi
+}
+
+BASE_DIR=`find_maven_basedir "$(pwd)"`
+if [ -z "$BASE_DIR" ]; then
+ exit 1;
+fi
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found .mvn/wrapper/maven-wrapper.jar"
+ fi
+else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
+ fi
+ if [ -n "$MVNW_REPOURL" ]; then
+ jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ else
+ jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ fi
+ while IFS="=" read key value; do
+ case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
+ esac
+ done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Downloading from: $jarUrl"
+ fi
+ wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
+ if $cygwin; then
+ wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
+ fi
+
+ if command -v wget > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found wget ... using wget"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ else
+ wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ fi
+ elif command -v curl > /dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found curl ... using curl"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ curl -o "$wrapperJarPath" "$jarUrl" -f
+ else
+ curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
+ fi
+
+ else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Falling back to using Java to download"
+ fi
+ javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ # For Cygwin, switch paths to Windows format before running javac
+ if $cygwin; then
+ javaClass=`cygpath --path --windows "$javaClass"`
+ fi
+ if [ -e "$javaClass" ]; then
+ if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Compiling MavenWrapperDownloader.java ..."
+ fi
+ # Compiling the Java class
+ ("$JAVA_HOME/bin/javac" "$javaClass")
+ fi
+ if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ # Running the downloader
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Running MavenWrapperDownloader.java ..."
+ fi
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+if [ "$MVNW_VERBOSE" = true ]; then
+ echo $MAVEN_PROJECTBASEDIR
+fi
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=`cygpath --path --windows "$M2_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ $MAVEN_DEBUG_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.home=${M2_HOME}" \
+ "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
--- /dev/null
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM https://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
+if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+ IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Found %WRAPPER_JAR%
+ )
+) else (
+ if not "%MVNW_REPOURL%" == "" (
+ SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
+ )
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Couldn't find %WRAPPER_JAR%, downloading it ...
+ echo Downloading from: %DOWNLOAD_URL%
+ )
+
+ powershell -Command "&{"^
+ "$webclient = new-object System.Net.WebClient;"^
+ "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+ "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+ "}"^
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
+ "}"
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Finished downloading %WRAPPER_JAR%
+ )
+)
+@REM End of extension
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% ^
+ %JVM_CONFIG_MAVEN_PROPS% ^
+ %MAVEN_OPTS% ^
+ %MAVEN_DEBUG_OPTS% ^
+ -classpath %WRAPPER_JAR% ^
+ "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
+ %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
+if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%"=="on" pause
+
+if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
+
+cmd /C exit /B %ERROR_CODE%
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.6.1</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>hu.user</groupId>
+ <artifactId>TrainCli</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>TrainCli</name>
+ <description>TrainCli</description>
+ <properties>
+ <java.version>11</java.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-amqp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-quartz</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/javax.annotation/javax.annotation-api -->
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.3.2</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
--- /dev/null
+package hu.user.traincli;
+
+public class ParametersExistYetException extends RuntimeException{
+
+ private String errorMessage;
+
+ public ParametersExistYetException(String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
--- /dev/null
+package hu.user.traincli;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+import hu.user.traincli.dto.EncodingAndFeatureEngineeringParameters;
+import hu.user.traincli.dto.EstimatorIdentifiers;
+import hu.user.traincli.dto.FitParameters;
+import hu.user.traincli.dto.TrainTaskParameters;
+import hu.user.traincli.service.JsonConverter;
+import hu.user.traincli.service.TrainClient;
+import hu.user.traincli.status.TrainStatus;
+
+@SpringBootApplication
+public class TrainCliApplication implements CommandLineRunner {
+
+ private static final String GET_CPU = "get_cpu";
+ private static final String GET_SCHEMAS = "get_schemas";
+ private static final String GET_TABLES = "get_tables";
+ private static final String SCHEMA_NAME = "schema_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String GET_TABLE_PROPERTY = "get_table_property";
+ private static final String GET_RECORDS = "get_records";
+ private static final String LIMIT = "limit";
+ private static final String OFFSET = "offset";
+ private static final String NEW_E_AND_FE_PARAMETERS = "new_e_and_fe_parameters";
+ private static final String GET_E_AND_FE_PARAMETERS = "get_e_and_fe_parameters";
+ private static final String GET_FEATURE_SELECTOR = "get_feature_selector";
+ private static final String GET_SAMPLER = "get_sampler";
+ private static final String GET_SCALER = "get_scaler";
+ private static final String GET_MODEL = "get_model";
+ private static final String NEW_TRAIN_TASK = "new_train_task";
+ private static final String GET_TRAIN_TASK = "get_train_task";
+ private static final String FIT = "fit";
+ private static final String GET_METRICS = "get_metrics";
+ private static final String GET_ESTIMATOR = "get_estimator";
+ private static final String ESTIMATOR_ID = "estimator_id";
+ private static final String ID = "id";
+
+ @Autowired TrainClient trainClient;
+
+ public static void main(String[] args) {
+ SpringApplication.run(TrainCliApplication.class, args);
+ }
+
+ @Override
+ public void run(String... args) throws Exception {
+ TrainStatus trainStatus = trainClient.testLiveness();
+ if (trainStatus == TrainStatus.NOT_ACCESSIBLE) {
+ System.out.println("Train module is not accessible");
+ System.exit(0);
+ } else if (trainStatus == TrainStatus.RESPONSE_TIMEOUT) {
+ System.out.println("Train's response time is to long");
+ System.exit(0);
+ } else if (trainStatus == TrainStatus.OK) {
+ System.out.println("Train module is accessible");
+ }
+ Map<String, String> queryParameters = new HashMap<>();
+ Integer id = null;
+ String filePath = null;
+ String jsonFileName = null;
+ String command = args[0];
+ switch (command) {
+ case GET_CPU:
+ System.out.println("Available CPU core number: " + trainClient.getCpuCoreNumber());
+ System.exit(0);
+ case GET_SCHEMAS:
+ String[] schemas = trainClient.getSchemas();
+ System.out.println("Schemas: ");
+ Arrays.stream(schemas).forEach(System.out::println);
+ System.exit(0);
+ case GET_TABLES:
+ if (args.length < 2) {
+ System.out.println("Schema name is missing");
+ System.exit(0);
+ } else {
+ String schemaName = args[1];
+ String[] tables = trainClient.getTables(Map.of(SCHEMA_NAME, schemaName));
+ System.out.println("Tables is schema: ");
+ Arrays.stream(tables).forEach(System.out::println);
+ System.exit(0);
+ }
+ System.exit(0);
+ case GET_TABLE_PROPERTY:
+ if (args.length < 2) {
+ System.out.println("Schema name is missing");
+ } else if (args.length < 3) {
+ System.out.println("Table name is missing");
+ } else {
+ String schema_name = args[1];
+ String table_name = args[2];
+ System.out.println("Table properties: ");
+ System.out.println(
+ trainClient.getTableProperties(Map.of(SCHEMA_NAME, schema_name, TABLE_NAME, table_name)));
+ }
+ System.exit(0);
+ case GET_RECORDS:
+ String schemaName = null;
+ String tableName = null;
+ Integer limit = null;
+ Integer offset = null;
+ if (args.length < 2) {
+ System.out.println("Schema name is missing");
+ System.exit(0);
+ } else {
+ schemaName = args[1];
+ }
+
+ if (args.length < 3) {
+ System.out.println("Table name is missing");
+ System.exit(0);
+ } else {
+ tableName = args[2];
+ }
+
+ if (args.length < 4) {
+ System.out.println("Limit is missing");
+ } else {
+ try {
+ limit = Integer.parseInt(args[3]);
+ } catch (NumberFormatException e) {
+ System.out.println("The limit parameter isn't integer");
+ System.exit(0);
+ }
+ }
+ if (args.length < 5) {
+ System.out.println("Offset is missing");
+ } else {
+ try {
+ offset = Integer.parseInt(args[4]);
+ } catch (NumberFormatException e) {
+ System.out.println("The offset parameter isn't integer");
+ System.exit(0);
+ }
+ }
+ queryParameters.put(SCHEMA_NAME, schemaName);
+ queryParameters.put(TABLE_NAME, tableName);
+ if (limit != null) {
+ queryParameters.put(LIMIT, String.valueOf(limit));
+ }
+ if (offset != null) {
+ queryParameters.put(OFFSET, String.valueOf(offset));
+ }
+ System.out.println(trainClient.getRecords(queryParameters));
+ System.exit(0);
+ case NEW_E_AND_FE_PARAMETERS:
+ if (args.length < 2) {
+ System.out.println("Path of JSON file is missing");
+ System.exit(0);
+ } else {
+ filePath = args[1];
+ }
+ if (args.length < 3) {
+ System.out.println("JSON file name is missing");
+ System.exit(0);
+ } else {
+ jsonFileName = args[2];
+ }
+ try {
+ EncodingAndFeatureEngineeringParameters encodingAndFeatureEngineeringParameters =
+ JsonConverter.jsonToJava(filePath, jsonFileName, EncodingAndFeatureEngineeringParameters.class);
+ String response =
+ trainClient.addEncodingAndFeatureEngineeringParameters(encodingAndFeatureEngineeringParameters);
+ System.out.println("Planned encoding and feature engineering id: " + response);
+ } catch (ParametersExistYetException exception) {
+ System.out.println(exception.getErrorMessage());
+ System.exit(0);
+ } catch (RuntimeException exception) {
+ System.out.println(exception.getMessage());
+ System.exit(0);
+ }
+ System.exit(0);
+ case GET_E_AND_FE_PARAMETERS:
+ if (args.length > 1) {
+ try {
+ id = Integer.parseInt(args[1]);
+ } catch (NumberFormatException e) {
+ System.out.println("The id parameter isn't integer");
+ System.exit(0);
+ }
+ if (id != null) {
+ queryParameters.put("id", String.valueOf(id));
+ }
+ }
+ System.out.println(trainClient.getEncodingAndFeatureEngineeringParameters(queryParameters));
+ System.exit(0);
+ case GET_FEATURE_SELECTOR:
+ System.out.println(trainClient.getFeatureSelectors());
+ System.exit(0);
+ case GET_SAMPLER:
+ System.out.println(trainClient.getSamplers());
+ System.exit(0);
+ case GET_SCALER:
+ System.out.println(trainClient.getScalers());
+ System.exit(0);
+ case GET_MODEL:
+ System.out.println(trainClient.getModels());
+ System.exit(0);
+ case NEW_TRAIN_TASK:
+ if (args.length < 2) {
+ System.out.println("Path is missing");
+ System.exit(0);
+ } else {
+ filePath = args[1];
+ }
+ if (args.length < 3) {
+ System.out.println("JSON file name is missing");
+ System.exit(0);
+ } else {
+ jsonFileName = args[2];
+ }
+ try {
+ TrainTaskParameters trainTaskParameters =
+ JsonConverter.jsonToJava(filePath, jsonFileName, TrainTaskParameters.class);
+ System.out.println("Train task id: " + trainClient.addTrainTask(trainTaskParameters));
+ System.exit(0);
+ } catch (ParametersExistYetException exception) {
+ System.out.println(exception.getErrorMessage());
+ System.exit(0);
+ } catch (RuntimeException exception) {
+ System.out.println(exception.getMessage());
+ System.exit(0);
+ }
+ System.exit(0);
+ case GET_TRAIN_TASK:
+ if (args.length > 1) {
+ try {
+ id = Integer.parseInt(args[1]);
+ } catch (NumberFormatException e) {
+ System.out.println("The id parameter isn't integer");
+ break;
+ }
+ }
+ if (id != null) {
+ queryParameters.put("id", String.valueOf(id));
+ }
+ System.out.println(trainClient.getTrainTask(queryParameters));
+ System.exit(0);
+ case FIT:
+ if (args.length < 2) {
+ System.out.println("Path is missing");
+ System.exit(0);
+ } else {
+ filePath = args[1];
+ }
+ if (args.length < 3) {
+ System.out.println("JSON file name is missing");
+ System.exit(0);
+ } else {
+ jsonFileName = args[2];
+ }
+ try {
+ FitParameters fitParameters = JsonConverter.jsonToJava(filePath, jsonFileName, FitParameters.class);
+ trainClient.fit(fitParameters);
+ } catch (ParametersExistYetException exception) {
+ System.out.println(exception.getErrorMessage());
+ System.exit(0);
+ } catch (RuntimeException exception) {
+ System.out.println(exception.getMessage());
+ }
+ System.exit(0);
+ case GET_METRICS:
+ if (args.length > 1) {
+ try {
+ id = Integer.parseInt(args[1]);
+ } catch (NumberFormatException e) {
+ System.out.println("The id parameter isn't integer");
+ System.exit(0);
+ }
+ }
+ if (id != null) {
+ queryParameters.put(ID, String.valueOf(id));
+ }
+ List<Map<String, Double>> metrics = trainClient.getMetrics(queryParameters);
+ for (int i = 0; i < metrics.size(); i++) {
+ System.out.println("Metrics");
+ for (var item : metrics.get(i).entrySet()) {
+ System.out.println(item.getKey() + ": " + item.getValue());
+ }
+ }
+ System.exit(0);
+ case GET_ESTIMATOR:
+ if (args.length > 1) {
+ try {
+ id = Integer.parseInt(args[1]);
+ } catch (NumberFormatException e) {
+ System.out.println("The id parameter isn't integer");
+ System.exit(0);
+ }
+ }
+ if (id != null) {
+ queryParameters.put("ID", String.valueOf(id));
+ }
+ EstimatorIdentifiers[] estimatorIdentifiers = trainClient.getEstimatorIdentifiers();
+ Arrays.stream(estimatorIdentifiers)
+ .forEach(e -> System.out.println("id: " + e.getId() + "; train_task_id: " + e.getTrainTaskId()));
+ System.exit(0);
+ default:
+ System.out.println("Available commmands:");
+ System.out.println("Get CPU core number: get_cpu");
+ System.out.println("Get shemas: get_schemas");
+ System.out.println("Get tables of a given schema: get_tables <schema name>");
+ System.out.println("Get properties of a given table: get_table_property <schema name> <table name>");
+ System.out.println("Get records of a given table: get_records <schema name> <table name> <limit> <offset>");
+ System.out.println("Add new encoding and feature engineering parameters: new_e_and_fe_parameters <path> <json file name>");
+ System.out.println("Add new encoding and feature engineering parameters: new_e_and_fe_parameters <path> <json file name>");
+ System.out.println("Get existing encoding and feature engineering parameters: get_e_and_fe_parameters");
+ System.out.println("Get installed feature selectors: get_feature_selector");
+ System.out.println("Get installed samplers: get_sampler");
+ System.out.println("Get installed scalers: get_scaler");
+ System.out.println("Get installed models: get_model");
+ System.out.println("Add new train task: new_train_task");
+ System.out.println("get existing train tasks: get_train_task");
+ System.out.println("launch fit process: fit");
+ System.out.println("get metrics: get_metrics");
+ System.out.println("get estimators: get_estimator");
+ System.exit(0);
+
+
+
+
+
+
+
+ System.exit(0);
+ }
+ }
+}
--- /dev/null
+package hu.user.traincli.config;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
+import org.springframework.amqp.support.converter.MessageConverter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class RabbitMqConfig {
+
+ @Bean
+ public Queue queue() {
+ return new Queue("train");
+ }
+
+ @Bean
+ public MessageConverter messageConverter() {
+ return new Jackson2JsonMessageConverter();
+ }
+
+}
--- /dev/null
+package hu.user.traincli.config;
+
+import java.time.Duration;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+import hu.user.traincli.error.RestTemplateErrorHandler;
+
+@Configuration
+public class RestConfig {
+
+ @Bean
+ public RestTemplate getRestTemplate(RestTemplateBuilder builder) {
+ RestTemplate restTemplate = builder
+ .setReadTimeout(Duration.ofSeconds(7))
+ .build();
+ return restTemplate;
+
+ }
+}
--- /dev/null
+package hu.user.traincli.config;
+
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.quartz.QuartzProperties;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.quartz.SchedulerFactoryBean;
+
+@Configuration
+public class SchedulerConfig {
+ @Autowired
+ private ApplicationContext applicationContext;
+ @Bean
+ public SchedulerFactoryBean schedulerFactoryBean() {
+ SchedulerJobFactory jobFactory = new SchedulerJobFactory();
+ jobFactory.setApplicationContext(applicationContext);
+ SchedulerFactoryBean factory = new SchedulerFactoryBean();
+ factory.setOverwriteExistingJobs(true);
+ factory.setJobFactory(jobFactory);
+ return factory;
+ }
+}
--- /dev/null
+package hu.user.traincli.config;
+
+import org.quartz.spi.TriggerFiredBundle;
+import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.scheduling.quartz.SpringBeanJobFactory;
+
+public class SchedulerJobFactory extends SpringBeanJobFactory implements ApplicationContextAware {
+
+ private AutowireCapableBeanFactory beanFactory;
+
+ @Override
+ public void setApplicationContext(final ApplicationContext context) {
+ beanFactory = context.getAutowireCapableBeanFactory();
+ }
+
+ @Override
+ protected Object createJobInstance(final TriggerFiredBundle bundle) throws Exception {
+ final Object job = super.createJobInstance(bundle);
+ beanFactory.autowireBean(job);
+ return job;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+public class CommonResponse {
+
+ private int id;
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ return "CommonResponse{" + "id=" + id + '}';
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class CommonTrainParameters {
+
+ private Map<String, String> parameters = new HashMap<>();
+
+ public Map<String, String> getParameters() {
+ return parameters;
+ }
+
+ public void setParameters(Map<String, String> parameters) {
+ this.parameters = parameters;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class EncodingAndFeatureEngineeringParameters {
+
+ private int id;
+ private String schemaName;
+ private String tableName;
+ private String timeBaseFieldName;
+ private Map<String, String> encodingParameters = new HashMap<>();
+ private FeatureEnginneringParameters featureEnginneringParameters;
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ @JsonSetter("schema_name")
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ @JsonSetter("table_name")
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getTimeBaseFieldName() {
+ return timeBaseFieldName;
+ }
+
+ @JsonSetter("time_base_field_name")
+ public void setTimeBaseFieldName(String timeBaseFieldName) {
+ this.timeBaseFieldName = timeBaseFieldName;
+ }
+
+ public Map<String, String> getEncodingParameters() {
+ return encodingParameters;
+ }
+
+ @JsonSetter("encoding_parameters")
+ public void setEncodingParameters(Map<String, String> encodingParameters) {
+ this.encodingParameters = encodingParameters;
+ }
+
+ public FeatureEnginneringParameters getFeatureEnginneringParameters() {
+ return featureEnginneringParameters;
+ }
+
+ @JsonSetter("feature_engineering_parameters")
+ public void setFeatureEnginneringParameters(FeatureEnginneringParameters featureEnginneringParameters) {
+ this.featureEnginneringParameters = featureEnginneringParameters;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class EstimatorIdentifiers {
+
+ private int id;
+ private int trainTaskId;
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public int getTrainTaskId() {
+ return trainTaskId;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setTrainTaskId(int trainTaskId) {
+ this.trainTaskId = trainTaskId;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FeatureEnginneringParameters {
+
+ private String chosesFeatureField;
+ private int[] intervals;
+
+ public String getChosesFeatureField() {
+ return chosesFeatureField;
+ }
+
+ @JsonSetter("chosen_feature_field")
+ public void setChosesFeatureField(String chosesFeatureField) {
+ this.chosesFeatureField = chosesFeatureField;
+ }
+
+ public int[] getIntervals() {
+ return intervals;
+ }
+
+ public void setIntervals(int[] intervals) {
+ this.intervals = intervals;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FieldProperty {
+
+ private String name;
+ private String[] otherApplicableEncodingTypes;
+ private String type;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String[] getOtherApplicableEncodingTypes() {
+ return otherApplicableEncodingTypes;
+ }
+
+ @JsonSetter("other_applicable_encoding_types")
+ public void setOtherApplicableEncodingTypes(String[] otherApplicableEncodingTypes) {
+ this.otherApplicableEncodingTypes = otherApplicableEncodingTypes;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + "\n" + "name : " + name + "," + '\n' + "otherApplicableEncodingTypes : " + summaryzeOtherApplicableEncodingTypes(
+ otherApplicableEncodingTypes) + "," + "\n" + "type : " + type + "\n" + "}";
+ }
+
+ private String summaryzeOtherApplicableEncodingTypes(String[] types) {
+ return Arrays.stream(types).collect(Collectors.joining(", "));
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FitParameters {
+
+ private int trainTaskId;
+ private int usedCpuCore;
+
+ public int getTrainTaskId() {
+ return trainTaskId;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setTrainTaskId(int trainTaskId) {
+ this.trainTaskId = trainTaskId;
+ }
+
+ public int getUsedCpuCore() {
+ return usedCpuCore;
+ }
+
+ @JsonSetter("used_cpu_core")
+ public void setUsedCpuCore(int usedCpuCore) {
+ this.usedCpuCore = usedCpuCore;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FraudCandidate {
+
+ private int fraudNumber;
+ private String name;
+ private int NoFraudNumber;
+
+ public int getFraudNumber() {
+ return fraudNumber;
+ }
+
+ @JsonSetter("fraud_number")
+ public void setFraudNumber(int fraudNumber) {
+ this.fraudNumber = fraudNumber;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getNoFraudNumber() {
+ return NoFraudNumber;
+ }
+
+ @JsonSetter("no_fraud_number")
+ public void setNoFraudNumber(int noFraudNumber) {
+ NoFraudNumber = noFraudNumber;
+ }
+
+ @Override
+ public String toString() {
+ return "FraudCandidate: {\n" + "fraudNumber: " + fraudNumber + ",\n" + "name: '" + name + ",\n" + "NoFraudNumber: " + NoFraudNumber + "\n" + "}";
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class LivenessDto {
+
+ private String response;
+
+ public LivenessDto(@JsonProperty(value = "response",required = true) String response) {
+ this.response = response;
+ }
+
+ public String getResponse() {
+ return response;
+ }
+
+ public void setResponse(String response) {
+ this.response = response;
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class TableProperties {
+
+ private List<FieldProperty> fields = new ArrayList<>();
+ private List<FraudCandidate> fraudCandidates = new ArrayList<>();
+ private String primaryKey;
+ private int recordNumber;
+ private String lineSeparator = System.lineSeparator();
+
+ public List<FieldProperty> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<FieldProperty> fields) {
+ this.fields = fields;
+ }
+
+ public List<FraudCandidate> getFraudCandidates() {
+ return fraudCandidates;
+ }
+
+ @JsonSetter("fraud_candidates")
+ public void setFraudCandidates(List<FraudCandidate> fraudCandidates) {
+ this.fraudCandidates = fraudCandidates;
+ }
+
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ @JsonSetter("primary_key")
+ public void setPrimaryKey(String primaryKey) {
+ this.primaryKey = primaryKey;
+ }
+
+ public int getRecordNumber() {
+ return recordNumber;
+ }
+
+ @JsonSetter("record_number")
+ public void setRecordNumber(int recordNumber) {
+ this.recordNumber = recordNumber;
+ }
+
+ @Override
+ public String toString() {
+ return "TableProperties: {" + convertToString() + "\n}";
+ }
+
+ private String convertToString() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("fields: [");
+ stringBuilder.append(lineSeparator);
+ stringBuilder.append(fields.stream().map(Objects::toString).collect(Collectors.joining(",\n")));
+ stringBuilder.append(",");
+ stringBuilder.append(lineSeparator);
+ stringBuilder.append(fraudCandidates.stream().map(Object::toString).collect(Collectors.joining(",\n")));
+ stringBuilder.append(lineSeparator);
+ stringBuilder.append("]");
+ stringBuilder.append(",");
+ stringBuilder.append(lineSeparator);
+ stringBuilder.append("Primary key: ");
+ stringBuilder.append(primaryKey);
+ stringBuilder.append(",");
+ stringBuilder.append(lineSeparator);
+ stringBuilder.append("Record number: ");
+ stringBuilder.append(recordNumber);
+// stringBuilder.append(lineSeparator);
+ return stringBuilder.toString();
+ }
+}
--- /dev/null
+package hu.user.traincli.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class TrainTaskParameters {
+
+ private int plannedEncodingAndFeatureEngineeringId;
+ private int featureSelectorCode;
+ private int samplerCode;
+ private int scalerCode;
+ private int model_code;
+ private double test_size;
+ private String trainTaskName;
+ private String uniqueFieldName;
+
+ public int getPlannedEncodingAndFeatureEngineeringId() {
+ return plannedEncodingAndFeatureEngineeringId;
+ }
+
+ @JsonSetter("planned_encoding_and_feature_engineering_id")
+ public void setPlannedEncodingAndFeatureEngineeringId(int plannedEncodingAndFeatureEngineeringId) {
+ this.plannedEncodingAndFeatureEngineeringId = plannedEncodingAndFeatureEngineeringId;
+ }
+
+ public int getFeatureSelectorCode() {
+ return featureSelectorCode;
+ }
+
+ @JsonSetter("feature_selector_code")
+ public void setFeatureSelectorCode(int featureSelectorCode) {
+ this.featureSelectorCode = featureSelectorCode;
+ }
+
+ public int getSamplerCode() {
+ return samplerCode;
+ }
+
+ @JsonSetter("sampler_code")
+ public void setSamplerCode(int samplerCode) {
+ this.samplerCode = samplerCode;
+ }
+
+ public int getScalerCode() {
+ return scalerCode;
+ }
+
+ @JsonSetter("scaler_code")
+ public void setScalerCode(int scalerCode) {
+ this.scalerCode = scalerCode;
+ }
+
+ public int getModel_code() {
+ return model_code;
+ }
+
+ @JsonSetter("model_code")
+ public void setModel_code(int model_code) {
+ this.model_code = model_code;
+ }
+
+ public double getTest_size() {
+ return test_size;
+ }
+
+ @JsonSetter("test_size")
+ public void setTest_size(double test_size) {
+ this.test_size = test_size;
+ }
+
+ public String getTrainTaskName() {
+ return trainTaskName;
+ }
+
+ @JsonSetter("train_task_name")
+ public void setTrainTaskName(String trainTaskName) {
+ this.trainTaskName = trainTaskName;
+ }
+
+ public String getUniqueFieldName() {
+ return uniqueFieldName;
+ }
+
+ @JsonSetter("unique_field_name")
+ public void setUniqueFieldName(String uniqueFieldName) {
+ this.uniqueFieldName = uniqueFieldName;
+ }
+
+ @Override
+ public String toString() {
+ return "TrainTaskParameters{" + "plannedEncodingAndFeatureEngineeringId=" + plannedEncodingAndFeatureEngineeringId + ", featureSelectorCode=" + featureSelectorCode + ", samplerCode=" + samplerCode + ", scalerCode=" + scalerCode + ", model_code=" + model_code + ", test_size=" + test_size + ", trainTaskName='" + trainTaskName + '\'' + ", uniqueFieldName='" + uniqueFieldName + '\'' + '}';
+ }
+}
--- /dev/null
+package hu.user.traincli.error;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.web.client.DefaultResponseErrorHandler;
+
+public class RestTemplateErrorHandler extends DefaultResponseErrorHandler {
+
+ @Override
+ protected void handleError(ClientHttpResponse response, HttpStatus statusCode) throws IOException {
+ byte[] responseBody = response.getBody().readAllBytes();
+ try {
+ Reader reader = new InputStreamReader(new ByteArrayInputStream(responseBody), Charset.defaultCharset());
+ CharBuffer buffer = CharBuffer.allocate(200);
+ reader.read(buffer);
+ reader.close();
+ buffer.flip();
+ System.out.println(buffer.toString());
+ }catch(IOException ex){
+ System.out.println("Exception in handle error");;
+ }
+ }
+ }
--- /dev/null
+package hu.user.traincli.event;
+
+import org.springframework.context.ApplicationEvent;
+
+import hu.user.traincli.status.TrainStatus;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@Getter
+public class TrainLiveCheckEvent extends ApplicationEvent {
+
+ private TrainStatus trainStatus;
+
+ public TrainLiveCheckEvent(Object source, TrainStatus trainStatus) {
+ super(source);
+ this.trainStatus = trainStatus;
+ }
+}
--- /dev/null
+package hu.user.traincli.event;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class TrainTaskEvent {
+
+ private int id;
+ private String method;
+ private String status;
+
+ public int getId() {
+ return id;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public String getMethod() {
+ return method;
+ }
+
+ public void setMethod(String method) {
+ this.method = method;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+}
--- /dev/null
+package hu.user.traincli.listenner;
+
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import hu.user.traincli.event.TrainTaskEvent;
+import hu.user.traincli.service.ProgressPresenter;
+import hu.user.traincli.train.ProgressName;
+import hu.user.traincli.train.ProgressState;
+import hu.user.traincli.train.ProgressStatus;
+import lombok.AllArgsConstructor;
+
+@Service
+@AllArgsConstructor
+public class TrainQueueListener {
+
+ @Autowired
+ private ProgressPresenter progressPresenter;
+
+ private final static String ENCODING = "ENCODING";
+ private final static String FEATURE_ENGINEERING = "FEATURE_ENGINEERING";
+ private final static String SAMPLING = "SAMPLING";
+ private final static String FITTING = "FITTING";
+ private final static String ESTIMATOR_PERSISTING = "ESTIMATOR_PERSISTING";
+ private final static String METRIC_PERSISTING = "METRIC_PERSISTING";
+ private final static String PREVIOUSLY_DONE = "PREVIOUSLY_DONE";
+ private final static String STARTED = "STARTED";
+ private final static String FINISHED = "FINISHED";
+
+
+ @RabbitListener(queues = "train")
+ public void receiveEvent(TrainTaskEvent event) {
+ ProgressState progressState = setProgressState(event);
+ progressPresenter.present(progressState);
+ System.out.println(
+ "Train task id: " + event.getId() + " ; method: " + event.getMethod() + " ; status: " + event.getStatus());
+
+ }
+
+ private ProgressState setProgressState(TrainTaskEvent event) {
+ ProgressName progressName = null;
+ ProgressStatus progressStatus = null;
+ switch (event.getMethod()) {
+ case ENCODING:
+ progressName = ProgressName.ENCODING;
+ break;
+ case FEATURE_ENGINEERING:
+ progressName = ProgressName.FEATURE_ENGINEERING;
+ break;
+ case SAMPLING:
+ progressName = ProgressName.SAMPLING;
+ break;
+ case FITTING:
+ progressName = ProgressName.FITTING;
+ break;
+ case ESTIMATOR_PERSISTING:
+ progressName = ProgressName.ESTIMATOR_PERSISTING;
+ break;
+ case METRIC_PERSISTING:
+ progressName = ProgressName.METRIC_PERSISTING;
+ break;
+ }
+ switch (event.getStatus()) {
+ case PREVIOUSLY_DONE:
+ progressStatus = ProgressStatus.PREVIOUSLY_DONE;
+ break;
+ case STARTED:
+ progressStatus = ProgressStatus.STARTED;
+ break;
+ case FINISHED:
+ progressStatus = ProgressStatus.FINISHED;
+ break;
+ }
+ ProgressState progressState = new ProgressState(progressName, progressStatus);
+ return progressState;
+ }
+}
+
+
+
+
+
+
+
+
+
--- /dev/null
+package hu.user.traincli.scheduled;
+
+import java.text.ParseException;
+import java.util.Date;
+
+import org.quartz.CronTrigger;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.SimpleTrigger;
+import org.springframework.context.ApplicationContext;
+import org.springframework.scheduling.quartz.CronTriggerFactoryBean;
+import org.springframework.scheduling.quartz.JobDetailFactoryBean;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.scheduling.quartz.SimpleTriggerFactoryBean;
+import org.springframework.stereotype.Component;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+public class JobScheduleCreator {
+
+ public JobDetail createJob(Class<? extends QuartzJobBean> jobClass, boolean isDurable, ApplicationContext context,
+ String jobName, JobDataMap jobDataMap ) {
+ JobDetailFactoryBean factoryBean = new JobDetailFactoryBean();
+ factoryBean.setJobClass(jobClass);
+ factoryBean.setDurability(isDurable);
+ factoryBean.setApplicationContext(context);
+ factoryBean.setName(jobName);
+ factoryBean.setJobDataMap(jobDataMap);
+ factoryBean.afterPropertiesSet();
+ return factoryBean.getObject();
+ }
+
+ public SimpleTrigger createSimpleTrigger(String triggerName, Date startTime, Long repeatTime) {
+ SimpleTriggerFactoryBean factoryBean = new SimpleTriggerFactoryBean();
+ factoryBean.setName(triggerName);
+ factoryBean.setStartTime(startTime);
+ factoryBean.setRepeatInterval(repeatTime);
+ factoryBean.setRepeatCount(SimpleTrigger.REPEAT_INDEFINITELY);
+ factoryBean.afterPropertiesSet();
+ SimpleTrigger simpleTrigger= factoryBean.getObject();
+ return simpleTrigger;
+ }
+}
--- /dev/null
+package hu.user.traincli.scheduled.job;
+
+import java.util.List;
+
+import org.quartz.DisallowConcurrentExecution;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.quartz.PersistJobDataAfterExecution;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+
+@PersistJobDataAfterExecution
+@DisallowConcurrentExecution
+public class ShowProgressJob extends QuartzJobBean {
+
+ public ShowProgressJob() {
+ }
+
+ public static final String EXECUTION_COUNT = "count";
+ public static final String FIRST_RUN = "first_run";
+
+ private List<String> characters = List.of("|","/", "-", "\\");
+
+ @Override
+ public void executeInternal(JobExecutionContext context) throws JobExecutionException {
+ JobDataMap data = context.getJobDetail().getJobDataMap();
+ int count = data.getInt(EXECUTION_COUNT);
+ boolean firstRun = data.getBoolean(FIRST_RUN);
+ if (!firstRun) {
+ System.out.print("\b");
+ }
+ int orderNumber = count % 4;
+ String actualString = characters.get(orderNumber);
+ System.out.print(actualString);
+ count++;
+ firstRun = false;
+ data.put(EXECUTION_COUNT, count);
+ data.put(FIRST_RUN, firstRun);
+ }
+}
--- /dev/null
+package hu.user.traincli.scheduled.job;
+
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.stereotype.Service;
+
+import hu.user.traincli.event.TrainLiveCheckEvent;
+import hu.user.traincli.service.TrainClient;
+import hu.user.traincli.status.TrainStatus;
+
+@Service
+public class TrainLivenessChecker extends QuartzJobBean {
+
+ @Autowired private TrainClient trainClient;
+
+ @Autowired private ApplicationEventPublisher applicationEventPublisher;
+
+ @Override
+ protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
+ checkIfTrainIsWorking();
+ }
+
+ private void checkIfTrainIsWorking() {
+ TrainStatus trainStatus = trainClient.testLiveness();
+ if (trainStatus == TrainStatus.NOT_ACCESSIBLE) {
+ applicationEventPublisher.publishEvent(new TrainLiveCheckEvent("Train", TrainStatus.NOT_ACCESSIBLE));
+ } else if (trainStatus == TrainStatus.RESPONSE_TIMEOUT) {
+ applicationEventPublisher.publishEvent(new TrainLiveCheckEvent("Train", TrainStatus.RESPONSE_TIMEOUT));
+ }
+ }
+}
--- /dev/null
+package hu.user.traincli.scheduled.jobinfo;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+@ToString
+@Getter
+@Setter
+public class SchedulerJobInfo {
+ private String jobName;
+ private String jobGroup;
+ private String jobClass;
+ private Long repeatTime;
+}
--- /dev/null
+package hu.user.traincli.scheduled.liveness;
+
+import java.util.List;
+
+import org.quartz.DisallowConcurrentExecution;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.quartz.PersistJobDataAfterExecution;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.stereotype.Service;
+
+import hu.user.traincli.service.TrainClient;
+import hu.user.traincli.status.TrainStatus;
+
+@PersistJobDataAfterExecution
+@DisallowConcurrentExecution
+@Service
+public class LivenessCheck extends QuartzJobBean {
+
+ @Autowired
+ private TrainClient trainClient;
+ public LivenessCheck() {
+ }
+
+ private List<String> characters = List.of("|","/", "-", "\\");
+
+ @Override
+ public void executeInternal(JobExecutionContext context) throws JobExecutionException {
+ TrainStatus trainStatus = trainClient.testLiveness();
+ if (trainStatus != TrainStatus.OK) {
+
+ }
+ }
+}
--- /dev/null
+package hu.user.traincli.scheduled.progress;
+
+import java.util.List;
+
+import org.quartz.DisallowConcurrentExecution;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.quartz.PersistJobDataAfterExecution;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+
+@PersistJobDataAfterExecution
+@DisallowConcurrentExecution
+public class ShowProgress extends QuartzJobBean {
+
+ public ShowProgress() {
+ }
+
+ public static final String EXECUTION_COUNT = "count";
+ public static final String FIRST_RUN = "first_run";
+
+ private List<String> characters = List.of("|","/", "-", "\\");
+
+ @Override
+ public void executeInternal(JobExecutionContext context) throws JobExecutionException {
+ JobDataMap data = context.getJobDetail().getJobDataMap();
+ int count = data.getInt(EXECUTION_COUNT);
+ boolean firstRun = data.getBoolean(FIRST_RUN);
+ if (!firstRun) {
+ System.out.print("\b");
+ }
+ int orderNumber = count % 4;
+ String actualString = characters.get(orderNumber);
+ System.out.print(actualString);
+ count++;
+ firstRun = false;
+ data.put(EXECUTION_COUNT, count);
+ data.put(FIRST_RUN, firstRun);
+ }
+}
--- /dev/null
+package hu.user.traincli.service;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class JsonConverter {
+
+ public static <T> T jsonToJava(String filePath, String fileName, Class<T> classType) {
+
+ T t = null;
+ String inputString;
+ Path path = Path.of(filePath + "/" + fileName);
+
+ try {
+ inputString = Files.readString(path);
+ } catch (IOException e) {
+ throw new RuntimeException("JSON file not found or can not be loaded");
+ }
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+ try {
+ t = mapper.readValue(inputString, classType);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException("JSON file can not be deserialized");
+ }
+ return t;
+ }
+}
--- /dev/null
+package hu.user.traincli.service;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationListener;
+import org.springframework.stereotype.Service;
+
+import hu.user.traincli.event.TrainLiveCheckEvent;
+import hu.user.traincli.status.TrainStatus;
+import hu.user.traincli.train.ProgressName;
+import hu.user.traincli.train.ProgressState;
+
+@Service
+public class ProgressPresenter implements ApplicationListener<TrainLiveCheckEvent> {
+
+ @Autowired
+ private SchedulerJobService schedulerJobService;
+
+ public void present(ProgressState progressState) {
+
+ switch (progressState.getProgressStatus()) {
+ case STARTED:
+ if (progressState.getProgressName().equals(ProgressName.ENCODING)) {
+ schedulerJobService.startScheduler();
+ } else {
+ schedulerJobService.resumeShowProgress();
+ }
+ System.out.println(progressState.toString());
+ System.out.print("In Progress: ");
+ break;
+ case FINISHED:
+// if (progressState.getProgressName().equals(ProgressName.METRIC_PERSISTING)) {
+// schedulerJobService.stopScheduler();
+// }
+ schedulerJobService.pauseShowProgress();
+ System.out.println();
+ System.out.println(progressState.toString());
+ break;
+ case PREVIOUSLY_DONE:
+ System.out.println(progressState.toString());
+ break;
+ }
+ }
+ @Override
+ public void onApplicationEvent(TrainLiveCheckEvent event) {
+ if (event.getTrainStatus().equals(TrainStatus.NOT_ACCESSIBLE)) {
+ System.out.println("Train module isn't responding");
+ schedulerJobService.pauseShowProgress();
+ schedulerJobService.pauseTrainLivenessCheck();
+ } else if (event.getTrainStatus().equals(TrainStatus.RESPONSE_TIMEOUT)) {
+ System.out.println("Train module isn't responding in time");
+ schedulerJobService.pauseShowProgress();
+ schedulerJobService.pauseTrainLivenessCheck();
+ }
+ }
+}
--- /dev/null
+package hu.user.traincli.service;
+
+import static hu.user.traincli.scheduled.job.ShowProgressJob.EXECUTION_COUNT;
+import static hu.user.traincli.scheduled.job.ShowProgressJob.FIRST_RUN;
+
+import java.time.Instant;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobKey;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.SchedulerMetaData;
+import org.quartz.SimpleTrigger;
+import org.quartz.Trigger;
+import org.quartz.TriggerKey;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.ApplicationContext;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.scheduling.quartz.SchedulerFactoryBean;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+import hu.user.traincli.scheduled.JobScheduleCreator;
+import hu.user.traincli.scheduled.job.ShowProgressJob;
+import hu.user.traincli.scheduled.job.TrainLivenessChecker;
+import hu.user.traincli.scheduled.jobinfo.SchedulerJobInfo;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Transactional
+@Service
+public class SchedulerJobService {
+
+ @Value("${showProgressRepeatTime}")
+ private long showProgressRepeatTime;
+ @Value("${trainLivenessRepeatTime}")
+ private long trainLivenessRepeatTime;
+
+ private static final String SHOW_PROGRESS_JOB = "showProgressJob";
+ private static final String SHOW_PROGRESS_TRIGGER = "showProgressTrigger";
+ private static final String TRAIN_LIVENESS_CHECKER_JOB = "trainLivenessCheckerJob";
+ private static final String TRAIN_LIVENESS_CHECKER_TRIGGER = "trainLivenessCheckerTrigger";
+ @Autowired private Scheduler scheduler;
+
+ @Autowired private SchedulerFactoryBean schedulerFactoryBean;
+
+ @Autowired private ApplicationContext context;
+
+ @Autowired private JobScheduleCreator scheduleCreator;
+
+ private JobDetail showProgressJobDetail;
+ private JobDetail trainLivenessCheckerJobDetail;
+
+ private SimpleTrigger showProgressTrigger;
+ private SimpleTrigger trainLivenessTrigger;
+
+ @PostConstruct
+ private void initialize() {
+ JobDataMap showProgressDataMap = new JobDataMap(Map.of(EXECUTION_COUNT, 0, FIRST_RUN, true));
+ showProgressJobDetail =
+ scheduleCreator.createJob(ShowProgressJob.class, true, context, SHOW_PROGRESS_JOB, showProgressDataMap);
+ trainLivenessCheckerJobDetail =
+ scheduleCreator.createJob(TrainLivenessChecker.class, true, context, TRAIN_LIVENESS_CHECKER_JOB,
+ new JobDataMap());
+ Date now = Date.from(Instant.now());
+ showProgressTrigger = scheduleCreator.createSimpleTrigger(SHOW_PROGRESS_TRIGGER, now,
+ showProgressRepeatTime);
+ trainLivenessTrigger = scheduleCreator.createSimpleTrigger(TRAIN_LIVENESS_CHECKER_TRIGGER, now,
+ trainLivenessRepeatTime);
+ scheduler = schedulerFactoryBean.getScheduler();
+ try {
+ scheduler.scheduleJob(showProgressJobDetail, showProgressTrigger);
+ scheduler.scheduleJob(trainLivenessCheckerJobDetail, trainLivenessTrigger);
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void startScheduler() {
+ try {
+ scheduler.start();
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void pauseShowProgress() {
+ try {
+ scheduler.pauseTrigger(showProgressTrigger.getKey());
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void pauseTrainLivenessCheck() {
+ try {
+ scheduler.pauseTrigger(trainLivenessTrigger.getKey());
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void resumeShowProgress() {
+ try {
+ scheduler.resumeTrigger(showProgressTrigger.getKey());
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void resumeTrainLivenessCheck() {
+ try {
+ scheduler.resumeTrigger(trainLivenessTrigger.getKey());
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void stopScheduler() {
+ try {
+ scheduler.shutdown();
+ } catch (SchedulerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+// public boolean pauseJob(SchedulerJobInfo jobInfo) {
+// try {
+// SchedulerJobInfo getJobInfo = schedulerRepository.findByJobName(jobInfo.getJobName());
+// getJobInfo.setJobStatus("PAUSED");
+// schedulerRepository.save(getJobInfo);
+// schedulerFactoryBean.getScheduler().pauseJob(new JobKey(jobInfo.getJobName(), jobInfo.getJobGroup()));
+// log.info(">>>>> jobName = [" + jobInfo.getJobName() + "]" + " paused.");
+// return true;
+// } catch (SchedulerException e) {
+// log.error("Failed to pause job - {}", jobInfo.getJobName(), e);
+// return false;
+// }
+// }
+
+// public boolean resumeJob(SchedulerJobInfo jobInfo) {
+// try {
+// SchedulerJobInfo getJobInfo = schedulerRepository.findByJobName(jobInfo.getJobName());
+// getJobInfo.setJobStatus("RESUMED");
+// schedulerRepository.save(getJobInfo);
+// schedulerFactoryBean.getScheduler().resumeJob(new JobKey(jobInfo.getJobName(), jobInfo.getJobGroup()));
+// log.info(">>>>> jobName = [" + jobInfo.getJobName() + "]" + " resumed.");
+// return true;
+// } catch (SchedulerException e) {
+// log.error("Failed to resume job - {}", jobInfo.getJobName(), e);
+// return false;
+// }
+// }
+
+// public boolean startJobNow(SchedulerJobInfo jobInfo) {
+// try {
+// SchedulerJobInfo getJobInfo = schedulerRepository.findByJobName(jobInfo.getJobName());
+// getJobInfo.setJobStatus("SCHEDULED & STARTED");
+// schedulerRepository.save(getJobInfo);
+// schedulerFactoryBean.getScheduler().triggerJob(new JobKey(jobInfo.getJobName(), jobInfo.getJobGroup()));
+// log.info(">>>>> jobName = [" + jobInfo.getJobName() + "]" + " scheduled and started now.");
+// return true;
+// } catch (SchedulerException e) {
+// log.error("Failed to start new job - {}", jobInfo.getJobName(), e);
+// return false;
+// }
+// }
+}
--- /dev/null
+package hu.user.traincli.service;
+
+import java.lang.reflect.ParameterizedType;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.ResourceAccessException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import hu.user.traincli.ParametersExistYetException;
+import hu.user.traincli.dto.CommonResponse;
+import hu.user.traincli.dto.CommonTrainParameters;
+import hu.user.traincli.dto.EncodingAndFeatureEngineeringParameters;
+import hu.user.traincli.dto.EstimatorIdentifiers;
+import hu.user.traincli.dto.FitParameters;
+import hu.user.traincli.dto.LivenessDto;
+import hu.user.traincli.dto.TableProperties;
+import hu.user.traincli.dto.TrainTaskParameters;
+import hu.user.traincli.status.TrainStatus;
+import lombok.Getter;
+import lombok.Setter;
+
+@Service
+@Getter
+@Setter
+public class
+TrainClient {
+
+ private static final String OK = "OK";
+ private static final String ENCODING_AND_FEATURE_ENGINEERING_PATH = "/encoding_and_feature_engineering_plan";
+ private static final String TRAIN_TASK_PATH = "/train_task";
+ private static final String FIT_PATH = "/fit";
+ private static final String SCHEMA_PATH = "/schema";
+ private static final String TABLE_PATH = "/table";
+ private static final String RECORD_PATH = "/record";
+ private static final String METRICS_PATH = "/metrics";
+ private static final String CPU_PATH = "/cpu";
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String LIVENESS_PATH = "/liveness";
+ private static final String TABLE_INFO_PATH="/table_info";
+
+
+
+ @Value("${trainUrl}") String trainUrl;
+ @Autowired private RestTemplate restTemplate;
+// private String trainUrl;
+
+ public TrainStatus testLiveness() {
+ TrainStatus trainStatus = null;
+ String url = trainUrl + LIVENESS_PATH;
+ try {
+ LivenessDto livenessDto = restTemplate.getForEntity(url, LivenessDto.class).getBody();
+ if (livenessDto.getResponse().equals(OK)) {
+ trainStatus = TrainStatus.OK;
+ }
+ } catch (ResourceAccessException exception) {
+ if (exception.getCause() instanceof ConnectException) {
+ trainStatus = TrainStatus.NOT_ACCESSIBLE;
+ }
+ if (exception.getCause() instanceof SocketTimeoutException) {
+ //TO-DO logging
+ trainStatus = TrainStatus.RESPONSE_TIMEOUT;
+ }
+ }
+ return trainStatus;
+ }
+
+ public int getCpuCoreNumber() {
+ String url = trainUrl + CPU_PATH;
+ int coreNumber = restTemplate.getForEntity(url, Integer.class).getBody();
+ return coreNumber;
+ }
+
+ public String[] getSchemas() {
+ String url = trainUrl + SCHEMA_PATH;
+ String[] schemas = restTemplate.getForEntity(url, String[].class).getBody();
+ return schemas;
+ }
+
+ public String[] getTables(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TABLE_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public TableProperties getTableProperties(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TABLE_INFO_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<TableProperties> tableProperties =
+ restTemplate.getForEntity(urlTemplate, TableProperties.class, params);
+ TableProperties result = tableProperties.getBody();
+ return result;
+ }
+
+ public String getRecords(Map<String, String> queryParamValueByName) {
+ String url = trainUrl +RECORD_PATH ;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+ public String getEncodingAndFeatureEngineeringParameters(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + ENCODING_AND_FEATURE_ENGINEERING_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+ public String getTrainTask(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TRAIN_TASK_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+// public EncodingAndFeatureEngineeringParameters getEncodingAndFeatureEngineeringParameters() {
+//
+// }
+
+ public Map<String, String> getFeatureSelectors() {
+ return useCommmonQuery("/available_feature_selector");
+ }
+
+ public Map<String, String> getSamplers() {
+ return useCommmonQuery("/available_sampler");
+ }
+
+ public Map<String, String> getScalers() {
+ return useCommmonQuery("/available_scaler");
+ }
+
+ public Map<String, String> getModels() {
+ return useCommmonQuery("/available_model");
+ }
+
+ public Map<String, String> useCommmonQuery(String path) {
+ String url = trainUrl + path;
+ Map<String,String> parameters = restTemplate.getForEntity(url, Map.class).getBody();
+ return parameters;
+ }
+
+ public EstimatorIdentifiers[] getEstimatorIdentifiers() {
+ String url = trainUrl + "/estimator";
+ EstimatorIdentifiers[] estimatorIdentifiers = restTemplate.getForEntity(url, EstimatorIdentifiers[].class).getBody();
+ return estimatorIdentifiers;
+ }
+
+ public List<Map<String, Double>> getMetrics(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + METRICS_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+
+ ParameterizedTypeReference<List<Map<String,Double>>> typeReference= new ParameterizedTypeReference<>() {};
+ List<Map<String, Double>> metrics = exchangeAsList(urlTemplate, typeReference, params);
+ return metrics;
+ }
+
+ public String addEncodingAndFeatureEngineeringParameters(EncodingAndFeatureEngineeringParameters parameters) {
+ return addParameters(trainUrl,parameters, ENCODING_AND_FEATURE_ENGINEERING_PATH);
+ }
+
+ public String addTrainTask(TrainTaskParameters trainTaskParameters) {
+ return addParameters(trainUrl,trainTaskParameters, TRAIN_TASK_PATH);
+ }
+
+ public void fit(FitParameters parameters) {
+ addParameters(trainUrl,parameters, FIT_PATH);
+ }
+
+ public <T> String addParameters(String trainUrl,T parameters, String path) {
+ String url = trainUrl + path;
+ try {
+ ResponseEntity<String> responseFromTrainModule =
+ restTemplate.postForEntity(url, parameters, String .class);
+ return responseFromTrainModule.getBody();
+ } catch (HttpClientErrorException httpClientErrorException) {
+ String errorMessage = httpClientErrorException.getResponseBodyAsString();
+ throw new ParametersExistYetException(errorMessage);
+ }
+ }
+
+ private <T> List<T> exchangeAsList(String uri, ParameterizedTypeReference<List<T>> responseType, Map<String,String> uriVariables) {
+ return restTemplate.exchange(uri, HttpMethod.GET, null, responseType,uriVariables).getBody();
+ }
+
+}
+
--- /dev/null
+package hu.user.traincli.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.web.util.UriComponentsBuilder;
+
+public class UrlBuilder {
+
+ private String url;
+ private Map<String, String> queryParams = new HashMap<>();
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Map<String, String> getQueryParams() {
+ return queryParams;
+ }
+
+ public void build(String url, Map<String, String> queryValueByParamName) {
+ List<String> paramNames = new ArrayList<>();
+ List<String> paramValues = new ArrayList<>();
+ for (var entry : queryValueByParamName.entrySet()) {
+ paramNames.add(entry.getKey());
+ paramValues.add(entry.getValue());
+ }
+ buildUrl(url, paramNames);
+ fillQueryParams(paramNames, paramValues);
+ }
+
+ private void buildUrl(String url, List<String> queryParamNames) {
+ String urlTemplate;
+ UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromHttpUrl(url);
+ for (String queryParamName : queryParamNames) {
+ uriComponentsBuilder = uriComponentsBuilder.queryParam(queryParamName, "{" + queryParamName + "}");
+ }
+ this.url = uriComponentsBuilder.encode().toUriString();
+ }
+
+ private void fillQueryParams(List<String> paramNames, List<String> paramValuee) {
+ Map<String, String> params = new HashMap<>();
+ for (int i = 0; i < paramNames.size(); i++) {
+ params.put(paramNames.get(i), paramValuee.get(i));
+ }
+ this.queryParams = params;
+ }
+}
--- /dev/null
+package hu.user.traincli.status;
+
+public enum TrainStatus {
+ OK,
+ NOT_ACCESSIBLE,
+ RESPONSE_TIMEOUT
+}
--- /dev/null
+package hu.user.traincli.train;
+
+public enum ProgressName {
+
+ ENCODING,
+ FEATURE_ENGINEERING,
+ SAMPLING,
+ FITTING,
+ ESTIMATOR_PERSISTING,
+ METRIC_PERSISTING
+}
--- /dev/null
+package hu.user.traincli.train;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class ProgressState {
+
+ private ProgressName progressName;
+ private ProgressStatus progressStatus;
+
+ @Override
+ public String toString() {
+ return progressName + " " + progressStatus;
+ }
+}
+
--- /dev/null
+package hu.user.traincli.train;
+
+public enum ProgressStatus {
+
+ READY,
+ PREVIOUSLY_DONE,
+ STARTED,
+ FINISHED
+}
--- /dev/null
+server.port=8086
+trainUrl=http://localhost:8085
+#trainUrl=http://mkigui:8085
+showProgressRepeatTime=500
+trainLivenessRepeatTime=100
+
--- /dev/null
+{
+ "planned_encoding_and_feature_engineering_id":2,
+ "feature_selector_code":1,
+ "sampler_code":1,
+ "scaler_code":1,
+ "model_code":8,
+ "test_size": 0.2
+}
\ No newline at end of file
--- /dev/null
+package hu.user.traincli;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class TrainCliApplicationTests {
+
+ @Test
+ void contextLoads() {
+ }
+
+}
--- /dev/null
+FROM adoptopenjdk:14-jre-hotspot
+WORKDIR /opt/app
+COPY target/*.jar trainmanager.jar
+CMD ["java","-jar","trainmanager.jar"]
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.5.5</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>hu.user</groupId>
+ <artifactId>TrainManager</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>TrainManager</name>
+ <description>TrainManager</description>
+ <properties>
+ <java.version>11</java.version>
+ <zkspringboot.version>2.5.3</zkspringboot.version>
+ <springboot.version>2.5.3</springboot.version>
+ </properties>
+
+ <repositories> <!--add-->
+ <repository>
+ <id>spring-snapshots</id>
+ <name>Spring Snapshots</name>
+ <url>https://repo.spring.io/snapshot</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>spring-milestones</id>
+ <name>Spring Milestones</name>
+ <url>https://repo.spring.io/milestone</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>ZK CE</id>
+ <name>ZK CE Repository</name>
+ <url>http://mavensync.zkoss.org/maven2</url>
+ </repository>
+ <repository>
+ <id>ZK EVAL</id>
+ <name>ZK Evaluation Repository</name>
+ <url>http://mavensync.zkoss.org/eval</url>
+ </repository>
+ </repositories>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-amqp</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-devtools</artifactId>
+ <version>${springboot.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency> <!--add-->
+ <groupId>org.zkoss.zkspringboot</groupId>
+ <artifactId>zkspringboot-starter</artifactId>
+ <type>pom</type>
+ <version>${zkspringboot.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.zkoss.zk</groupId>
+ <artifactId>zkplus</artifactId>
+ <version>9.6.0-Eval</version>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>8.0.28</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.opencsv/opencsv -->
+ <dependency>
+ <groupId>com.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ <version>5.7.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
--- /dev/null
+package hu.user.trainmanager;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class TrainManagerApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(TrainManagerApplication.class, args);
+ }
+
+}
--- /dev/null
+package hu.user.trainmanager.component;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.annotation.ScopedProxyMode;
+import org.springframework.stereotype.Component;
+
+import hu.user.trainmanager.dto.ConfusionMatrixDto;
+import hu.user.trainmanager.dto.FitParameters;
+import hu.user.trainmanager.dto.TrainTaskParameters;
+import hu.user.trainmanager.dto.progress.TrainProgressScreenProperties;
+import hu.user.trainmanager.dto.progress.TrainProgressState;
+import hu.user.trainmanager.event.TrainTaskEvent;
+import hu.user.trainmanager.event.TrainTaskMethodState;
+import hu.user.trainmanager.service.TrainClient;
+import hu.user.trainmanager.status.TrainTaskState;
+import lombok.Getter;
+import lombok.Setter;
+
+@Component
+@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS)
+@Getter
+@Setter
+public class TrainProgressStateHolder {
+
+ private static final String TRAIN_TASK_ID = "train_task_id";
+ @Autowired private TrainClient trainClient;
+
+ public TrainProgressStateHolder(TrainClient trainClient) {
+ this.trainClient = trainClient;
+ }
+
+// private List<EstimatorAndTrainTaskIdentifiers> estimatorAndTrainTaskIdentifiers = new ArrayList<>();
+ private List<TrainProgressState> trainProgressStates = new ArrayList<>();
+ private Map<Integer, TrainTaskParameters> trainTaskParametersById = new HashMap<>();
+
+ private List<String> trainWorkCollection = new ArrayList<>();
+
+ private TrainProgressScreenProperties trainProgressScreenProperties;
+
+ private boolean trainInProgress;
+
+ private int usedCpuCoreNumber;
+
+ private int availableCpuCoreNumber;
+
+ private boolean stopped;
+
+ @PostConstruct
+ public void init() {
+ availableCpuCoreNumber = trainClient.getCpuCoreNumber();
+ trainTaskParametersById = trainClient.getTrainTask(Collections.emptyMap());
+// loadEstimatorAndTrainTaskIdentifiers();
+// List<Integer> fittedTrainTaskIds =
+// estimatorAndTrainTaskIdentifiers.stream().map(e -> e.getTrainTaskId()).collect(Collectors.toList());
+ // @formatter:off
+ trainProgressStates = trainTaskParametersById
+ .entrySet()
+ .stream()
+ .filter(e->e.getValue().getTrainTaskState().equals(TrainTaskState.STORED))
+ .map(e->new TrainProgressState(e.getKey(), e.getValue().getTrainTaskName(),TrainTaskMethodState.READY, TrainTaskMethodState.READY,
+ TrainTaskMethodState.READY, TrainTaskMethodState.READY, TrainTaskMethodState.READY,
+ TrainTaskMethodState.READY ,TrainTaskState.STORED,false))
+ .collect(Collectors.toList());
+ // @formatter:on
+ int allTrainTaskNumber = trainTaskParametersById.keySet().size();
+ int finishedTrainTaskNumber =
+ (int) trainTaskParametersById.entrySet().stream().filter(e->e.getValue().getTrainTaskState().equals(TrainTaskState.DONE)).count();
+ trainProgressScreenProperties =
+ new TrainProgressScreenProperties(allTrainTaskNumber, Optional.of(finishedTrainTaskNumber),
+ Optional.empty(), Optional.empty(), true, true, false);
+ }
+
+// public void loadEstimatorAndTrainTaskIdentifiers() {
+// estimatorAndTrainTaskIdentifiers = trainClient.getEstimatorIdentifiers();
+// }
+
+ public ConfusionMatrixDto getConfusionMatrixElements(int trainTaskId) {
+ return trainClient.getConfusionMatrixElements(Map.of(TRAIN_TASK_ID, String.valueOf(trainTaskId)));
+ }
+
+ public TrainProgressScreenProperties changeProgressStatus(TrainTaskEvent event) {
+ int currentTrainTaskId = event.getTrainTaskId();
+ Optional<TrainProgressState> optionalTrainProgress =
+ trainProgressStates.stream().filter(e -> e.getId() == currentTrainTaskId).findFirst();
+ if (optionalTrainProgress.isPresent()) {
+ TrainProgressState currentTrainProgressState = optionalTrainProgress.get();
+ TrainTaskMethodState trainTaskMethodState = event.getMethodState();
+ switch (event.getMethodName()) {
+ case ENCODING:
+ currentTrainProgressState.setEncodingState(trainTaskMethodState);
+ currentTrainProgressState.setTrainTaskState(TrainTaskState.IN_PROGRESS);
+ System.out.println("Changed encoding state");
+ break;
+ case FEATURE_ENGINEERING:
+ currentTrainProgressState.setFeatureEngineeringState(trainTaskMethodState);
+ System.out.println("Changed feature engineering state");
+ break;
+ case SAMPLING:
+ currentTrainProgressState.setSamplingProcessState(trainTaskMethodState);
+ System.out.println("Changed sampling state");
+ break;
+ case FITTING:
+ currentTrainProgressState.setFitProcessState(trainTaskMethodState);
+ System.out.println("Changed fitting state");
+ break;
+ case ESTIMATOR_PERSISTING:
+ currentTrainProgressState.setEstimatorPersistingState(trainTaskMethodState);
+ break;
+ case METRIC_PERSISTING:
+ currentTrainProgressState.setMetricsPersistingState(trainTaskMethodState);
+ currentTrainProgressState.setTrainTaskState(TrainTaskState.DONE);
+ incrementFinishedTrainTaskNumber();
+ eraseCurrentProgressedTrainTaskId();
+ break;
+ }
+ }
+ return trainProgressScreenProperties;
+ }
+
+ private void eraseCurrentProgressedTrainTaskId() {
+ trainProgressScreenProperties.setCurrentProgressedTrainTaskId(Optional.empty());
+ }
+
+ private void enableStopButton() {
+ trainProgressScreenProperties.setStopButtonDisabled(false);
+ }
+
+ private void incrementFinishedTrainTaskNumber() {
+ if (trainProgressScreenProperties.getFinishedTrainTaskNumber().isPresent()) {
+ trainProgressScreenProperties.setFinishedTrainTaskNumber(
+ Optional.of(trainProgressScreenProperties.getFinishedTrainTaskNumber().get() + 1));
+ } else {
+ trainProgressScreenProperties.setFinishedTrainTaskNumber(Optional.of(1));
+ }
+ }
+
+ public TrainProgressScreenProperties addTrainTaskToWorkCollection(String trainTaskid) {
+ if (!trainWorkCollection.contains(trainTaskid)) {
+ trainWorkCollection.add(trainTaskid);
+ if (trainProgressScreenProperties.getToBeDoneTrainTaskNumber().isPresent()) {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(
+ Optional.of(trainProgressScreenProperties.getToBeDoneTrainTaskNumber().get() + 1));
+ } else {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(Optional.of(1));
+ }
+ }
+ return trainProgressScreenProperties;
+ }
+
+ public TrainProgressScreenProperties removeTrainTaskFromWorkCollection(String trainTaskid) {
+ if (trainWorkCollection.contains(trainTaskid)) {
+ trainWorkCollection.remove(trainTaskid);
+ }
+ if (trainWorkCollection.isEmpty()) {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(Optional.empty());
+ } else {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(Optional.of(trainWorkCollection.size()));
+ }
+ return trainProgressScreenProperties;
+ }
+
+ public TrainProgressScreenProperties startTrainProcess() {
+ if (!trainWorkCollection.isEmpty()) {
+ trainInProgress = true;
+ disableAllAddButton();
+ startNextTrainTask();
+ }
+ return trainProgressScreenProperties;
+ }
+
+ private void disableAllAddButton() {
+ trainProgressStates.stream().forEach(e -> e.setAddButtonDisabled(true));
+ }
+
+ public TrainProgressScreenProperties startNextTrainTask() {
+ if (!trainWorkCollection.isEmpty() && !stopped) {
+ int nextTrainTaskId = Integer.parseInt(trainWorkCollection.remove(0));
+ trainProgressScreenProperties.setCurrentProgressedTrainTaskId(Optional.of(nextTrainTaskId));
+ refreshToBeDoneTrainTaskNumber();
+ startSingleTrainTask(nextTrainTaskId);
+ } else {
+ disableStopButton();
+ enableStartButton();
+ enableTrainTaskRefreshButton();
+ trainProgressScreenProperties.setCurrentProgressedTrainTaskId(Optional.empty());
+ }
+ return trainProgressScreenProperties;
+ }
+
+ private void disableStopButton() {
+ trainProgressScreenProperties.setStopButtonDisabled(true);
+ }
+
+ private void enableStartButton() {
+ trainProgressScreenProperties.setStartButtonDisabled(false);
+ }
+
+ private void enableTrainTaskRefreshButton() {
+ trainProgressScreenProperties.setTrainTaskRefreshButtonDisabled(false);
+ }
+
+ private void refreshToBeDoneTrainTaskNumber() {
+ if (trainWorkCollection.isEmpty()) {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(Optional.empty());
+ } else {
+ trainProgressScreenProperties.setToBeDoneTrainTaskNumber(Optional.of(trainWorkCollection.size()));
+ }
+ }
+
+ public void startSingleTrainTask(int trainTaskId) {
+ FitParameters fitParameters = new FitParameters(trainTaskId, usedCpuCoreNumber);
+ trainClient.fit(fitParameters);
+ }
+
+ public void refreshTrainTasks() {
+ init();
+ }
+
+ public void refreshAddButtonStates() {
+ for (TrainProgressState trainProgressState : trainProgressStates) {
+ if (trainProgressState.getTrainTaskState().equals(TrainTaskState.DONE)) {
+ trainProgressState.setAddButtonDisabled(true);
+ } else {
+ trainProgressState.setAddButtonDisabled(false);
+ }
+ }
+ }
+}
--- /dev/null
+package hu.user.trainmanager.config;\r
+\r
+import javax.annotation.PostConstruct;\r
+\r
+import org.slf4j.Logger;\r
+import org.slf4j.LoggerFactory;\r
+import org.springframework.context.annotation.Configuration;\r
+import org.springframework.context.annotation.Profile;\r
+import org.zkoss.lang.Library;\r
+import org.zkoss.zk.ui.WebApps;\r
+\r
+@Configuration\r
+@Profile("dev")\r
+public class DevelopmentConfig {\r
+ private static Logger logger = LoggerFactory.getLogger(DevelopmentConfig.class);\r
+\r
+ @PostConstruct\r
+ public void initDevelopmentProperties() throws Exception {\r
+ logger.info("**************************************************************");\r
+ logger.info("**** ZK-Springboot-Demo: development configuration active ****");\r
+ logger.info("**************************************************************");\r
+\r
+ //disable various caches to avoid server restarts\r
+ Library.setProperty("org.zkoss.zk.ZUML.cache", "false");\r
+ Library.setProperty("org.zkoss.zk.WPD.cache", "false");\r
+ Library.setProperty("org.zkoss.zk.WCS.cache", "false");\r
+ Library.setProperty("org.zkoss.web.classWebResource.cache", "false");\r
+ Library.setProperty("org.zkoss.util.label.cache", "false");\r
+\r
+ // enable non minified js\r
+ WebApps.getCurrent().getConfiguration().setDebugJS(true);\r
+\r
+ // enable for debugging MVVM commands and binding (very verbose)\r
+ Library.setProperty("org.zkoss.bind.DebuggerFactory.enable", "false");\r
+ }\r
+}\r
--- /dev/null
+package hu.user.trainmanager.config;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
+import org.springframework.amqp.support.converter.MessageConverter;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class RabbitMqConfig {
+
+ @Bean
+ public Queue queue() {
+ return new Queue("train");
+ }
+
+ @Bean
+ public MessageConverter messageConverter() {
+ return new Jackson2JsonMessageConverter();
+ }
+
+}
--- /dev/null
+package hu.user.trainmanager.config;
+
+import java.time.Duration;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+
+@Configuration
+public class RestConfig {
+
+ @Bean
+ public RestTemplate getRestTemplate(RestTemplateBuilder builder) {
+ RestTemplate restTemplate = builder
+ .setReadTimeout(Duration.ofSeconds(7))
+ .build();
+ return restTemplate;
+
+ }
+}
--- /dev/null
+package hu.user.trainmanager.controller;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.zkoss.bind.BindUtils;
+import org.zkoss.bind.GlobalCommandEvent;
+import org.zkoss.bind.annotation.DefaultGlobalCommand;
+import org.zkoss.bind.annotation.GlobalCommand;
+import org.zkoss.zk.ui.Component;
+import org.zkoss.zk.ui.Desktop;
+import org.zkoss.zk.ui.Execution;
+import org.zkoss.zk.ui.Executions;
+import org.zkoss.zk.ui.event.Event;
+import org.zkoss.zk.ui.event.EventListener;
+import org.zkoss.zk.ui.event.EventQueues;
+import org.zkoss.zk.ui.event.ForwardEvent;
+import org.zkoss.zk.ui.select.SelectorComposer;
+import org.zkoss.zk.ui.select.annotation.Listen;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.Wire;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.Button;
+import org.zkoss.zul.Combobox;
+import org.zkoss.zul.ListModelList;
+import org.zkoss.zul.Listbox;
+import org.zkoss.zul.Listcell;
+import org.zkoss.zul.Listitem;
+import org.zkoss.zul.Textbox;
+
+import hu.user.trainmanager.component.TrainProgressStateHolder;
+import hu.user.trainmanager.dto.progress.TrainProgressScreenProperties;
+import hu.user.trainmanager.dto.progress.TrainProgressState;
+import hu.user.trainmanager.event.TrainTaskEvent;
+import hu.user.trainmanager.event.TrainTaskMethodName;
+import hu.user.trainmanager.event.TrainTaskMethodState;
+import hu.user.trainmanager.listenner.TrainQueueListener;
+import hu.user.trainmanager.viewmodel.ProgressPresenter;
+import lombok.Getter;
+
+@Getter
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class TrainTaskController extends SelectorComposer<Component> implements ProgressPresenter {
+
+ private static final String ON_CLICK = "onClick";
+
+ @WireVariable private TrainProgressStateHolder trainProgressStateHolder;
+
+ @WireVariable private TrainQueueListener trainQueueListener;
+
+ @Wire private Listbox progressListbox;
+
+ @Wire private Button startButton;
+// @Wire private Button stopButton;
+
+ @Wire private Combobox cpuCoreComboBox;
+
+ @Wire private Button trainTasksRefreshButton;
+
+ @Wire private Textbox allTrainTaskNuber;
+ @Wire private Textbox finishedTrainTaskNumber;
+ @Wire private Textbox tobeDoneTrainTaskNumber;
+ @Wire private Textbox currentLyInProgressTrainTaskId;
+
+ private TrainProgressScreenProperties trainProgressScreenProperties;
+
+ private Desktop desktop;
+
+ private ListModelList<TrainProgressState> statesModel = new ListModelList<>();
+ private ListModelList<Integer> cpuCoreModel = new ListModelList<>();
+
+ @Override
+ public void doAfterCompose(Component comp) throws Exception {
+ super.doAfterCompose(comp);
+ trainQueueListener.addPresenter(this);
+ desktop = comp.getDesktop();
+ desktop.enableServerPush(true);
+ cpuCoreComboBox.setModel(new ListModelList<>(
+ IntStream.rangeClosed(1, trainProgressStateHolder.getAvailableCpuCoreNumber()).boxed()
+ .collect(Collectors.toList())));
+ cpuCoreComboBox.addEventListener("onChange", new CpuCoreComboboxChangeListener());
+ progressListbox.setModel(statesModel);
+ statesModel.addAll(new ListModelList<>(trainProgressStateHolder.getTrainProgressStates()));
+ trainProgressScreenProperties = trainProgressStateHolder.getTrainProgressScreenProperties();
+ startButton.setDisabled(trainProgressScreenProperties.isStartButtonDisabled());
+// stopButton.setDisabled(trainProgressScreenProperties.isStopButtonDisabled());
+ allTrainTaskNuber.setValue(String.valueOf(trainProgressScreenProperties.getAllTrainTaskNumber()));
+ trainProgressScreenProperties.getFinishedTrainTaskNumber()
+ .ifPresent(value -> finishedTrainTaskNumber.setValue(String.valueOf(value)));
+ trainProgressScreenProperties.getToBeDoneTrainTaskNumber()
+ .ifPresent(value -> tobeDoneTrainTaskNumber.setValue(String.valueOf(value)));
+ trainProgressScreenProperties.getCurrentProgressedTrainTaskId()
+ .ifPresent(value -> currentLyInProgressTrainTaskId.setValue(String.valueOf(value)));
+ }
+
+ @Listen("onClick = #startButton")
+ public void startTrainProgress() {
+ trainProgressScreenProperties = trainProgressStateHolder.startTrainProcess();
+ if (trainProgressScreenProperties.getToBeDoneTrainTaskNumber().isPresent()) {
+ tobeDoneTrainTaskNumber.setValue(
+ String.valueOf(trainProgressScreenProperties.getToBeDoneTrainTaskNumber().get()));
+ } else {
+ tobeDoneTrainTaskNumber.setValue(null);
+ }
+ currentLyInProgressTrainTaskId.setValue(
+ String.valueOf(trainProgressScreenProperties.getCurrentProgressedTrainTaskId().get()));
+ startButton.setDisabled(true);
+ trainTasksRefreshButton.setDisabled(true);
+ statesModel.clear();
+ statesModel.addAll(new ListModelList<>(trainProgressStateHolder.getTrainProgressStates()));
+ }
+
+
+ @Listen("onSelect = #cpuCoreComboBox")
+ public void selectCpuCore() {
+ int coreNumber = cpuCoreComboBox.getSelectedItem().getValue();
+ trainProgressStateHolder.setUsedCpuCoreNumber(coreNumber);
+ }
+ @DefaultGlobalCommand
+ public void automaticRefreshTrainTasks(Event event) {
+ System.out.println("Event: "+event.getName());
+ if (event instanceof GlobalCommand) {
+ if ("automaticRefreshTrainTasks".equals(((GlobalCommandEvent) event).getCommand())) {
+ System.out.println("automatic refresh");
+ refeshTrainTasks();
+
+ }
+ }
+ }
+ @Listen("onClick = #trainTasksRefreshButton ")
+ public void refeshTrainTasks() {
+ trainProgressStateHolder.refreshTrainTasks();
+ statesModel.clear();
+ statesModel.addAll(new ListModelList<>(trainProgressStateHolder.getTrainProgressStates()));
+ trainProgressScreenProperties = trainProgressStateHolder.getTrainProgressScreenProperties();
+ startButton.setDisabled(trainProgressScreenProperties.isStartButtonDisabled());
+ allTrainTaskNuber.setValue(String.valueOf(trainProgressScreenProperties.getAllTrainTaskNumber()));
+ trainProgressScreenProperties.getFinishedTrainTaskNumber()
+ .ifPresent(value -> finishedTrainTaskNumber.setValue(String.valueOf(value)));
+ trainProgressScreenProperties.getToBeDoneTrainTaskNumber()
+ .ifPresent(value -> tobeDoneTrainTaskNumber.setValue(String.valueOf(value)));
+ trainProgressScreenProperties.getCurrentProgressedTrainTaskId()
+ .ifPresent(value -> currentLyInProgressTrainTaskId.setValue(String.valueOf(value)));
+
+ }
+
+ @Listen("onShow = #progressListbox")
+ public void onShow(ForwardEvent forwardEvent) {
+ Button button = (Button) forwardEvent.getOrigin().getTarget();
+ Listitem listitem = (Listitem) button.getParent().getParent();
+ List<Listcell> listcells = listitem.getChildren();
+ String trainTaskid = listcells.get(0).getLabel();
+ System.out.println("id: " + trainTaskid);
+ }
+
+ @Listen("onAddButtonClick = #progressListbox")
+ public void onAddButtonClick(ForwardEvent forwardEvent) {
+ Button addTrainTaskButton = (Button) forwardEvent.getOrigin().getTarget();
+ addTrainTaskButton.setDisabled(true);
+ Listitem listitem = (Listitem) addTrainTaskButton.getParent().getParent();
+ List<Listcell> listcells = listitem.getChildren();
+ String trainTaskid = listcells.get(0).getLabel();
+ trainProgressScreenProperties = trainProgressStateHolder.addTrainTaskToWorkCollection(trainTaskid);
+ tobeDoneTrainTaskNumber.setValue(
+ String.valueOf(trainProgressScreenProperties.getToBeDoneTrainTaskNumber().get()));
+ if (trainProgressStateHolder.getUsedCpuCoreNumber() > 0) {
+ startButton.setDisabled(false);
+ }
+ trainTasksRefreshButton.setDisabled(true);
+ }
+
+ @Override
+ public void changeUi(TrainTaskEvent trainTaskEvent) {
+ Executions.schedule(desktop, new TriggeredListener(), new Event("triggered", progressListbox, trainTaskEvent));
+ }
+
+ class TriggeredListener implements EventListener {
+ @Override
+ public void onEvent(Event event) throws Exception {
+ TrainTaskEvent trainTaskEvent = (TrainTaskEvent) event.getData();
+ trainProgressScreenProperties = trainProgressStateHolder.changeProgressStatus(trainTaskEvent);
+ List<TrainProgressState> progressStates = trainProgressStateHolder.getTrainProgressStates();
+ statesModel.clear();
+ statesModel.addAll(new ListModelList<>(progressStates));
+ if (trainTaskEvent.getMethodName()
+ .equals(TrainTaskMethodName.METRIC_PERSISTING) && trainTaskEvent.getMethodState()
+ .equals(TrainTaskMethodState.FINISHED)) {
+ if (trainProgressStateHolder.getTrainWorkCollection().isEmpty()) {
+ trainTasksRefreshButton.setDisabled(false);
+ trainProgressStateHolder.refreshAddButtonStates();
+ statesModel.clear();
+ statesModel.addAll(new ListModelList<>(progressStates));
+ BindUtils.postGlobalCommand(null,null,"refreshMetrics",null);
+ }
+
+ finishedTrainTaskNumber.setValue(
+ String.valueOf(trainProgressScreenProperties.getFinishedTrainTaskNumber().get()));
+ trainProgressScreenProperties = trainProgressStateHolder.startNextTrainTask();
+ if (trainProgressScreenProperties.getCurrentProgressedTrainTaskId().isPresent()) {
+ currentLyInProgressTrainTaskId.setValue(
+ String.valueOf(trainProgressScreenProperties.getCurrentProgressedTrainTaskId().get()));
+ } else {
+ currentLyInProgressTrainTaskId.setValue(null);
+ }
+ if (trainProgressScreenProperties.getToBeDoneTrainTaskNumber().isPresent()) {
+ tobeDoneTrainTaskNumber.setValue(
+ String.valueOf(trainProgressScreenProperties.getToBeDoneTrainTaskNumber().get()));
+ } else {
+ tobeDoneTrainTaskNumber.setValue(null);
+ }
+ }
+ }
+ }
+
+ class CpuCoreComboboxChangeListener implements EventListener {
+ @Override
+ public void onEvent(Event event) throws Exception {
+ if (!trainProgressStateHolder.getTrainWorkCollection().isEmpty()) {
+ startButton.setDisabled(false);
+ } else {
+ startButton.setDisabled(true);
+ }
+ }
+ }
+
+}
+
+
--- /dev/null
+package hu.user.trainmanager.dto;
+
+public class CommonResponse {
+
+ private int id;
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ return "CommonResponse{" + "id=" + id + '}';
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class CommonTrainParameters {
+
+ private Map<String, String> parameters = new HashMap<>();
+
+ public Map<String, String> getParameters() {
+ return parameters;
+ }
+
+ public void setParameters(Map<String, String> parameters) {
+ this.parameters = parameters;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import org.springframework.beans.factory.annotation.Autowired;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class ConfusionMatrixDto {
+
+ private int TP;
+ private int TN;
+ private int FP;
+ private int FN;
+
+
+
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import java.io.File;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class DirectoryProperty {
+
+ private String displayName;
+ private File directory;
+
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+
+@Getter
+public class DownStreamErrorDto {
+
+ private int errorCode;
+ private String errorName;
+ private String errorMessage;
+
+ public DownStreamErrorDto(@JsonProperty(value = "error_code", required = true) int errorCode,
+ @JsonProperty(value = "error_name", required = true) String errorName,
+ @JsonProperty(value = "error_message", required = true) String errorMessage) {
+ this.errorCode = errorCode;
+ this.errorName = errorName;
+ this.errorMessage = errorMessage;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class EncodingAndFeatureEngineeringParameters {
+
+ private int id;
+ private String schemaName;
+ private String tableName;
+ private String timeBaseFieldName;
+ private Map<String, String> encodingParameters = new HashMap<>();
+ private FeatureEnginneringParameters featureEnginneringParameters;
+
+ public EncodingAndFeatureEngineeringParameters() {
+ }
+
+ public EncodingAndFeatureEngineeringParameters(
+ @JsonProperty(value = "schema_name", required = true) String schemaName,
+ @JsonProperty(value = "table_name", required = true) String tableName,
+ @JsonProperty(value = "time_base_field_name", required = true) String timeBaseFieldName,
+ @JsonProperty(value = "encoding_parameters", required = true) Map<String, String> encodingParameters,
+ @JsonProperty(value = "feature_engineering_parameters", required = false)
+ FeatureEnginneringParameters featureEnginneringParameters) {
+ this.schemaName = schemaName;
+ this.tableName = tableName;
+ this.timeBaseFieldName = timeBaseFieldName;
+ this.encodingParameters = encodingParameters;
+ this.featureEnginneringParameters = featureEnginneringParameters;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ @JsonSetter("schema_name")
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ @JsonSetter("table_name")
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getTimeBaseFieldName() {
+ return timeBaseFieldName;
+ }
+
+ @JsonSetter("time_base_field_name")
+ public void setTimeBaseFieldName(String timeBaseFieldName) {
+ this.timeBaseFieldName = timeBaseFieldName;
+ }
+
+ public Map<String, String> getEncodingParameters() {
+ return encodingParameters;
+ }
+
+ @JsonSetter("encoding_parameters")
+ public void setEncodingParameters(Map<String, String> encodingParameters) {
+ this.encodingParameters = encodingParameters;
+ }
+
+ public FeatureEnginneringParameters getFeatureEnginneringParameters() {
+ return featureEnginneringParameters;
+ }
+
+ @JsonSetter("feature_engineering_parameters")
+ public void setFeatureEnginneringParameters(FeatureEnginneringParameters featureEnginneringParameters) {
+ this.featureEnginneringParameters = featureEnginneringParameters;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class EstimatorAndTrainTaskIdentifiers {
+
+ private int id;
+ private int trainTaskId;
+
+ public EstimatorAndTrainTaskIdentifiers(int id, @JsonProperty(value = "train_task_id") int trainTaskId) {
+ this.id = id;
+ this.trainTaskId = trainTaskId;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public int getTrainTaskId() {
+ return trainTaskId;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setTrainTaskId(int trainTaskId) {
+ this.trainTaskId = trainTaskId;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FeatureEnginneringParameters {
+
+ private String chosenFeatureField;
+ private int[] intervals;
+
+ public FeatureEnginneringParameters(@JsonProperty(value = "chosen_feature_field",required = true) String chosenFeatureField,
+ int[] intervals) {
+ this.chosenFeatureField = chosenFeatureField;
+ this.intervals = intervals;
+ }
+
+ public String getChosenFeatureField() {
+ return chosenFeatureField;
+ }
+
+ @JsonSetter("chosen_feature_field")
+ public void setChosenFeatureField(String chosenFeatureField) {
+ this.chosenFeatureField = chosenFeatureField;
+ }
+
+ public int[] getIntervals() {
+ return intervals;
+ }
+
+ public void setIntervals(int[] intervals) {
+ this.intervals = intervals;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FieldProperty {
+
+ private String name;
+ private String[] otherApplicableEncodingTypes;
+ private String type;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String[] getOtherApplicableEncodingTypes() {
+ return otherApplicableEncodingTypes;
+ }
+
+ @JsonSetter("other_applicable_encoding_types")
+ public void setOtherApplicableEncodingTypes(String[] otherApplicableEncodingTypes) {
+ this.otherApplicableEncodingTypes = otherApplicableEncodingTypes;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import java.io.File;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class FileProperty {
+
+ private File file;
+ private String fileName;
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FitParameters {
+
+ private int trainTaskId;
+ private int usedCpuCore;
+
+ public FitParameters(@JsonProperty(value = "train_task_id", required = true) int trainTaskId,
+ @JsonProperty(value = "used_cpu_core", required = true) int usedCpuCore) {
+ this.trainTaskId = trainTaskId;
+ this.usedCpuCore = usedCpuCore;
+ }
+
+ public int getTrainTaskId() {
+ return trainTaskId;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setTrainTaskId(int trainTaskId) {
+ this.trainTaskId = trainTaskId;
+ }
+
+ public int getUsedCpuCore() {
+ return usedCpuCore;
+ }
+
+ @JsonSetter("used_cpu_core")
+ public void setUsedCpuCore(int usedCpuCore) {
+ this.usedCpuCore = usedCpuCore;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class FraudCandidate {
+
+ private int fraudNumber;
+ private String name;
+ private int NoFraudNumber;
+
+ public int getFraudNumber() {
+ return fraudNumber;
+ }
+
+ @JsonSetter("fraud_number")
+ public void setFraudNumber(int fraudNumber) {
+ this.fraudNumber = fraudNumber;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getNoFraudNumber() {
+ return NoFraudNumber;
+ }
+
+ @JsonSetter("no_fraud_number")
+ public void setNoFraudNumber(int noFraudNumber) {
+ NoFraudNumber = noFraudNumber;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class LivenessDto {
+
+ private String response;
+
+ public LivenessDto(@JsonProperty(value = "response",required = true) String response) {
+ this.response = response;
+ }
+
+ public String getResponse() {
+ return response;
+ }
+
+ public void setResponse(String response) {
+ this.response = response;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class TableProperties {
+
+ private List<FieldProperty> fields = new ArrayList<>();
+ private List<FraudCandidate> fraudCandidates = new ArrayList<>();
+ private String primaryKey;
+ private int recordNumber;
+
+ public List<FieldProperty> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<FieldProperty> fields) {
+ this.fields = fields;
+ }
+
+ public List<FraudCandidate> getFraudCandidates() {
+ return fraudCandidates;
+ }
+
+ @JsonSetter("fraud_candidates")
+ public void setFraudCandidates(List<FraudCandidate> fraudCandidates) {
+ this.fraudCandidates = fraudCandidates;
+ }
+
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ @JsonSetter("primary_key")
+ public void setPrimaryKey(String primaryKey) {
+ this.primaryKey = primaryKey;
+ }
+
+ public int getRecordNumber() {
+ return recordNumber;
+ }
+
+ @JsonSetter("record_number")
+ public void setRecordNumber(int recordNumber) {
+ this.recordNumber = recordNumber;
+ }
+
+ @Override
+ public String toString() {
+ return "TableProperties{" + "fields=" + fields + ", fraudCandidates=" + fraudCandidates + ", primaryKey='" + primaryKey + '\'' + ", recordNumber=" + recordNumber + '}';
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto;
+
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+import hu.user.trainmanager.status.TrainTaskState;
+
+public class TrainTaskParameters {
+ private String trainTaskName;
+ private String uniqueFieldName;
+ private int plannedEncodingAndFeatureEngineeringId;
+ private int featureSelectorCode;
+ private int samplerCode;
+ private int scalerCode;
+ private int model_code;
+ private double testSize;
+ private TrainTaskState trainTaskState;
+
+ public TrainTaskParameters() {
+ }
+
+ // @formatter:off
+ public TrainTaskParameters(
+ @JsonProperty(value = "train_task_name",required = true) String trainTaskName,
+ @JsonProperty(value = "unique_field_name",required = true) String uniqueFieldName,
+ @JsonProperty(value = "planned_encoding_and_feature_engineering_id", required = true) int plannedEncodingAndFeatureEngineeringId,
+ @JsonProperty(value = "feature_selector_code", required = true) int featureSelectorCode,
+ @JsonProperty(value = "sampler_code", required = true) int samplerCode,
+ @JsonProperty(value = "scaler_code", required = true) int scalerCode,
+ @JsonProperty(value = "model_code", required = true) int model_code,
+ @JsonProperty(value = "test_size", required = true) double testSize,
+ @JsonProperty(value = "process_status",required = true) TrainTaskState trainTaskState) {
+ this.trainTaskName = trainTaskName;
+ this.uniqueFieldName=uniqueFieldName;
+ this.plannedEncodingAndFeatureEngineeringId = plannedEncodingAndFeatureEngineeringId;
+ this.featureSelectorCode = featureSelectorCode;
+ this.samplerCode = samplerCode;
+ this.scalerCode = scalerCode;
+ this.model_code = model_code;
+ this.testSize = testSize;
+ this.trainTaskState = trainTaskState;
+ }
+ // @formatter:on
+ @JsonGetter("train_task_name")
+ public String getTrainTaskName() {
+ return trainTaskName;
+ }
+
+ @JsonSetter("train_task_name")
+ public void setTrainTaskName(String trainTaskName) {
+ this.trainTaskName = trainTaskName;
+ }
+
+ @JsonSetter("unique_field_name")
+ public void setUniqueFieldName(String uniqueFieldName) {
+ this.uniqueFieldName = uniqueFieldName;
+ }
+
+ @JsonGetter("unique_field_name")
+ public String getUniqueFieldName() {
+ return uniqueFieldName;
+ }
+
+ public int getPlannedEncodingAndFeatureEngineeringId() {
+ return plannedEncodingAndFeatureEngineeringId;
+ }
+
+ @JsonSetter("planned_encoding_and_feature_engineering_id")
+ public void setPlannedEncodingAndFeatureEngineeringId(int plannedEncodingAndFeatureEngineeringId) {
+ this.plannedEncodingAndFeatureEngineeringId = plannedEncodingAndFeatureEngineeringId;
+ }
+
+ public int getFeatureSelectorCode() {
+ return featureSelectorCode;
+ }
+
+ @JsonSetter("feature_selector_code")
+ public void setFeatureSelectorCode(int featureSelectorCode) {
+ this.featureSelectorCode = featureSelectorCode;
+ }
+
+ public int getSamplerCode() {
+ return samplerCode;
+ }
+
+ @JsonSetter("sampler_code")
+ public void setSamplerCode(int samplerCode) {
+ this.samplerCode = samplerCode;
+ }
+
+ public int getScalerCode() {
+ return scalerCode;
+ }
+
+ @JsonSetter("scaler_code")
+ public void setScalerCode(int scalerCode) {
+ this.scalerCode = scalerCode;
+ }
+
+ public int getModel_code() {
+ return model_code;
+ }
+
+ @JsonSetter("model_code")
+ public void setModel_code(int model_code) {
+ this.model_code = model_code;
+ }
+
+ public double getTestSize() {
+ return testSize;
+ }
+
+ @JsonSetter("test_size")
+ public void setTestSize(double testSize) {
+ this.testSize = testSize;
+ }
+
+ public TrainTaskState getTrainTaskState() {
+ return trainTaskState;
+ }
+
+ @JsonSetter("process_status")
+ public void setTrainTaskState(TrainTaskState trainTaskState) {
+ this.trainTaskState = trainTaskState;
+ }
+
+}
--- /dev/null
+package hu.user.trainmanager.dto.csv;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class CsvContentDto {
+
+ private List<String> headerElements = new ArrayList<>();
+ private List<List<String>> lines = new ArrayList<>();
+
+ public CsvContentDto(@JsonProperty(value = "header_elements", required = true) List<String> headerElements,
+ @JsonProperty(value = "lines", required = true) List<List<String>> lines) {
+ this.headerElements = headerElements;
+ this.lines = lines;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto.csv;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class CsvDataHolder {
+
+ private List<String> headerElements = new ArrayList<>();
+ private List<List<String>> contentLines = new ArrayList<>();
+}
--- /dev/null
+package hu.user.trainmanager.dto.csv;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class ReducingAndEvaulationParameterRequestDto {
+
+ public String schemaName;
+ private String tableName;
+ private String fraudTypeFieldName;
+ private int positiveFraudCaseNumber;
+ private int negatíveFraudCaseNumber;
+ private boolean dateSuffixToAppend;
+
+ // @formatter:off
+ public ReducingAndEvaulationParameterRequestDto(
+ @JsonProperty(value = "schema_name", required = true) String schemaName,
+ @JsonProperty(value = "table_name", required = true) String tableName,
+ @JsonProperty(value = "fraud_type_field_name", required = true) String fraudTypeFieldName,
+ @JsonProperty(value = "positive_fraud_case_number", required = true) int positiveFraudCaseNumber,
+ @JsonProperty(value = "negative_fraud_case_number", required = true) int negatíveFraudCaseNumber,
+ @JsonProperty(value = "date_suffix_to_append",required = true) boolean dateSuffixToAppend) {
+ // @formatter:on
+ this.schemaName = schemaName;
+ this.tableName = tableName;
+ this.fraudTypeFieldName = fraudTypeFieldName;
+ this.positiveFraudCaseNumber = positiveFraudCaseNumber;
+ this.negatíveFraudCaseNumber = negatíveFraudCaseNumber;
+ this.dateSuffixToAppend = dateSuffixToAppend;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.dto.csv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class ReducingAndEvaulationParameterResponseDto {
+
+ private String plannedCsvFileName;
+ private Map<Integer, Integer> plannedCsvContent = new HashMap<>();
+
+ // @formatter:off
+ public ReducingAndEvaulationParameterResponseDto(
+ @JsonProperty(value = "planned_csv_file_name",required = true) String plannedCsvFileName,
+ @JsonProperty(value = "planned_csv_content",required = true) Map<Integer, Integer> plannedCsvContent) {
+ this.plannedCsvFileName = plannedCsvFileName;
+ this.plannedCsvContent = plannedCsvContent;
+ }
+ // @formatter:on
+}
--- /dev/null
+package hu.user.trainmanager.dto.display;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class EstimatorMetricsDto {
+ private int id;
+ private int tp;
+ private int tn;
+ private int fp;
+ private int fn;
+ private double sensitivity;
+ private double specifity;
+ private double accuracy;
+ private double balancedAccuracy;
+ private double prec;
+ private double recall;
+ private double ppv;
+ private double npv;
+ private double fnr;
+ private double fpr;
+ private double fdr;
+ private double _for;
+ private double f1;
+ private double f05;
+ private double f2;
+ private double mcc;
+ private double rocauc;
+ private double youden;
+
+}
--- /dev/null
+package hu.user.trainmanager.dto.display;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class FieldNameAndEncodingType {
+
+ private String fieldName;
+ private String encodingType;
+}
--- /dev/null
+package hu.user.trainmanager.dto.progress;
+
+import java.util.Optional;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+public class TrainProgressScreenProperties {
+
+ private int allTrainTaskNumber;
+ private Optional<Integer> finishedTrainTaskNumber;
+ private Optional<Integer> toBeDoneTrainTaskNumber;
+ private Optional<Integer> currentProgressedTrainTaskId;
+ private boolean startButtonDisabled;
+ private boolean stopButtonDisabled;
+ private boolean trainTaskRefreshButtonDisabled;
+}
--- /dev/null
+package hu.user.trainmanager.dto.progress;
+
+import hu.user.trainmanager.event.TrainTaskMethodState;
+import hu.user.trainmanager.status.TrainTaskState;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@AllArgsConstructor
+@Getter
+@Setter
+public class TrainProgressState {
+
+ private int id;
+ private String trainTaskName;
+ private TrainTaskMethodState encodingState;
+ private TrainTaskMethodState featureEngineeringState;
+ private TrainTaskMethodState samplingProcessState;
+ private TrainTaskMethodState fitProcessState;
+ private TrainTaskMethodState estimatorPersistingState;
+ private TrainTaskMethodState metricsPersistingState;
+ private TrainTaskState trainTaskState;
+ private boolean addButtonDisabled;
+
+}
--- /dev/null
+package hu.user.trainmanager.event;
+
+import org.springframework.context.ApplicationEvent;
+
+public class MessageArrivedFromTrainModule extends ApplicationEvent {
+ public MessageArrivedFromTrainModule(Object source) {
+ super(source);
+ }
+}
--- /dev/null
+package hu.user.trainmanager.event;
+
+import org.springframework.context.ApplicationEvent;
+
+import hu.user.trainmanager.status.TrainStatus;
+import lombok.Getter;
+
+@Getter
+public class TrainLiveCheckEvent extends ApplicationEvent {
+
+ private TrainStatus trainStatus;
+
+ public TrainLiveCheckEvent(Object source, TrainStatus trainStatus) {
+ super(source);
+ this.trainStatus = trainStatus;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.event;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+public class TrainTaskEvent {
+
+ private int trainTaskId;
+ private TrainTaskMethodName methodName;
+ private TrainTaskMethodState methodState;
+
+ public TrainTaskEvent(@JsonProperty(value = "train_task_id", required = true) int trainTaskId,
+ @JsonProperty(value = "method", required = true) TrainTaskMethodName methodName,
+ @JsonProperty(value = "status", required = true) TrainTaskMethodState methodState) {
+ this.trainTaskId = trainTaskId;
+ this.methodName = methodName;
+ this.methodState = methodState;
+ }
+
+ public int getTrainTaskId() {
+ return trainTaskId;
+ }
+
+ @JsonSetter("train_task_id")
+ public void setTrainTaskId(int trainTaskId) {
+ this.trainTaskId = trainTaskId;
+ }
+
+ public TrainTaskMethodName getMethodName() {
+ return methodName;
+ }
+
+ public void setMethodName(TrainTaskMethodName methodName) {
+ this.methodName = methodName;
+ }
+
+ public TrainTaskMethodState getMethodState() {
+ return methodState;
+ }
+
+ public void setMethodState(TrainTaskMethodState methodState) {
+ this.methodState = methodState;
+ }
+
+ @Override
+ public String toString() {
+ return "TrainTaskEvent{" + "trainTaskId=" + trainTaskId + ", methodName=" + methodName + ", methodState=" + methodState + '}';
+ }
+}
--- /dev/null
+package hu.user.trainmanager.event;
+
+public enum TrainTaskMethodName {
+
+ // @formatter:off
+ ENCODING,
+ FEATURE_ENGINEERING,
+ SAMPLING,
+ FITTING,
+ ESTIMATOR_PERSISTING,
+ METRIC_PERSISTING;
+ // @formatter:on
+
+}
--- /dev/null
+package hu.user.trainmanager.event;
+
+public enum TrainTaskMethodState {
+ READY, PREVIOUSLY_DONE, STARTED, FINISHED
+}
--- /dev/null
+package hu.user.trainmanager.exception;
+
+public class ParametersExistYetException extends RuntimeException{
+
+ private String errorMessage;
+
+ public ParametersExistYetException(String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.listenner;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.annotation.ScopedProxyMode;
+import org.springframework.stereotype.Service;
+
+import hu.user.trainmanager.component.TrainProgressStateHolder;
+import hu.user.trainmanager.event.TrainTaskEvent;
+import hu.user.trainmanager.viewmodel.ProgressPresenter;
+
+@Service
+@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS)
+public class TrainQueueListener {
+ @Autowired private TrainProgressStateHolder trainProgressStateHolder;
+
+ private List<ProgressPresenter> presenters = new ArrayList<>();
+
+ public void addPresenter(ProgressPresenter presenter) {
+ presenters.clear();
+ presenters.add(presenter);
+ }
+
+ @RabbitListener(queues = "train")
+ public void receiveEvent(TrainTaskEvent event) {
+ System.out.println("TrainQueueListener hashCode in receiveEvent: " + this.hashCode());
+ System.out.println(event);
+// trainProgressStateHolder.changeProgressStatus(event);
+// Events.postEvent(new Event("onTrigger"));
+ presenters.stream().forEach(e -> e.changeUi(event));
+ }
+}
+
+
+
+
+
+
+
+
+
--- /dev/null
+package hu.user.trainmanager.service;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.opencsv.CSVWriter;
+
+import hu.user.trainmanager.dto.csv.CsvContentDto;
+import hu.user.trainmanager.dto.csv.CsvDataHolder;
+import lombok.AllArgsConstructor;
+
+@Service
+@AllArgsConstructor
+public class CsvHandler {
+
+ private TrainClient trainClient;
+
+ public void createEvaulationCsvFile(String fullPath, String fileName, Map<Integer, Integer> csvContent) {
+ List<String> headerElements = new ArrayList<>(List.of("id", "fraud"));
+ List<List<String>> contentLines =
+ csvContent.entrySet().stream().map(e -> createLine(e)).collect(Collectors.toList());
+ CsvDataHolder csvDataHolder = new CsvDataHolder(headerElements, contentLines);
+ String filePath = fullPath + "/" + fileName;
+ filePath += ".csv";
+ writeDataLineByLine(filePath, csvDataHolder);
+ }
+
+ private List<String> createLine(Map.Entry<Integer, Integer> e) {
+ return List.of(String.valueOf(e.getKey()), String.valueOf(e.getValue()));
+ }
+
+ public void createTrainTaskCsvFile(String fullPath, String fileName, boolean dateTimeSuffixEnabled) {
+ CsvContentDto csvContentDto = trainClient.getTrainTaskCsvContent();
+ CsvDataHolder csvDataHolder = new CsvDataHolder(csvContentDto.getHeaderElements(), csvContentDto.getLines());
+ String filePath = fullPath + "/" + fileName;
+ if (dateTimeSuffixEnabled) {
+ LocalDateTime now = LocalDateTime.now();
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_hhmmss");
+ String formattedDateTime = now.format(formatter);
+ filePath += "_" + formattedDateTime;
+ }
+ filePath += ".csv";
+ writeDataLineByLine(filePath, csvDataHolder);
+ }
+
+ public void createMetricsCsvFile(String fullPath, String fileName, boolean dateTimeSuffixEnabled) {
+ CsvContentDto csvContentDto = trainClient.getMetricsCsvContent();
+ CsvDataHolder csvDataHolder = new CsvDataHolder(csvContentDto.getHeaderElements(), csvContentDto.getLines());
+ String filePath = fullPath + "/" + fileName;
+ if (dateTimeSuffixEnabled) {
+ LocalDateTime now = LocalDateTime.now();
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_hhmmss");
+ String formattedDateTime = now.format(formatter);
+ filePath += "_" + formattedDateTime;
+ }
+ filePath += ".csv";
+ writeDataLineByLine(filePath, csvDataHolder);
+ }
+
+ public void writeDataLineByLine(String filePath, CsvDataHolder csvDataHolder) {
+ File file = new File(filePath);
+ List<String> headerElements = csvDataHolder.getHeaderElements();
+ List<List<String>> contentLines = csvDataHolder.getContentLines();
+ try (FileWriter outputfile = new FileWriter(file)) {
+ CSVWriter writer = new CSVWriter(outputfile);
+ String[] header = headerElements.stream().toArray(String[]::new);
+ writer.writeNext(header);
+ for (List<String> lineItems : contentLines) {
+ String[] items = lineItems.stream().toArray(String[]::new);
+ writer.writeNext(items);
+ }
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+}
--- /dev/null
+package hu.user.trainmanager.service;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import hu.user.trainmanager.dto.csv.CsvDataHolder;
+import lombok.AllArgsConstructor;
+
+@Service
+@AllArgsConstructor
+public class MetricsCsvDataBuilder {
+
+ private static final String ESTIMATOR_ID = "estimator_id";
+ private static final String TRAIN_TASK_NAME = "train_task_name";
+ private static final List<String> METRIC_NAMES = new ArrayList<>(
+ Arrays.asList("TP", "FP", "TN", "FN", "sensitivity", "specificity", "accuracy", "balanced_accuracy", "prec",
+ "recall", "PPV", "NPV", "FNR", "FPR", "FDR", "FOR", "f1", "f0.5", "f2", "MCC", "ROCAUC", "Youdens_statistic"));
+
+ private TrainClient trainClient;
+
+ public CsvDataHolder getCsvContent() {
+ List<String> headerElements = new ArrayList<>();
+ List<List<String>> lines = new ArrayList<>();
+ List<Map<String, Double>> metricsFromTrainModule = trainClient.getMetrics(Collections.emptyMap());
+ List<Integer> estimatorIds =
+ metricsFromTrainModule.get(0).entrySet().stream().filter(e -> e.getKey().equals(ESTIMATOR_ID))
+ .map(e -> e.getValue().intValue()).collect(Collectors.toList());
+ List<String> estimatorNames = estimatorIds.stream()
+ .map(e -> trainClient.getTrainTaskNameByEstimatorId(Map.of(ESTIMATOR_ID, String.valueOf(e))))
+ .collect(Collectors.toList());
+ headerElements.add(TRAIN_TASK_NAME);
+ headerElements.addAll(METRIC_NAMES);
+
+ int dataLineNumber = metricsFromTrainModule.size();
+ for (int i = 0; i < dataLineNumber; i++) {
+ List<String> singleLine = new ArrayList<>();
+ singleLine.add(estimatorNames.get(i));
+ Map<String, Double> singleMetrics = metricsFromTrainModule.get(i);
+ for (String metricName : METRIC_NAMES) {
+ singleLine.add(String.valueOf(singleMetrics.get(metricName)));
+ }
+ lines.add(singleLine);
+ }
+ CsvDataHolder csvDataHolder = new CsvDataHolder(headerElements, lines);
+ return csvDataHolder;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.service;
+
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Scope;
+import org.springframework.context.annotation.ScopedProxyMode;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.ResourceAccessException;
+import org.springframework.web.client.RestTemplate;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import hu.user.trainmanager.dto.ConfusionMatrixDto;
+import hu.user.trainmanager.dto.DownStreamErrorDto;
+import hu.user.trainmanager.dto.EncodingAndFeatureEngineeringParameters;
+import hu.user.trainmanager.dto.EstimatorAndTrainTaskIdentifiers;
+import hu.user.trainmanager.dto.FitParameters;
+import hu.user.trainmanager.dto.LivenessDto;
+import hu.user.trainmanager.dto.TableProperties;
+import hu.user.trainmanager.dto.TrainTaskParameters;
+import hu.user.trainmanager.dto.csv.CsvContentDto;
+import hu.user.trainmanager.dto.csv.ReducingAndEvaulationParameterRequestDto;
+import hu.user.trainmanager.dto.csv.ReducingAndEvaulationParameterResponseDto;
+import hu.user.trainmanager.status.TrainModuleStatus;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS)
+@Slf4j
+public class TrainClient {
+ private static final String OK = "OK";
+ private static final String ENCODING_AND_FEATURE_ENGINEERING_PATH = "/encoding_and_feature_engineering_plan";
+ private static final String TRAIN_TASK_PATH = "/train_task";
+ private static final String FIT_PATH = "/fit";
+ private static final String SCHEMA_PATH = "/schema";
+ private static final String TABLE_PATH = "/table";
+ private static final String FIELD_NAME_AND_TYPE_PATH = "/field_name_and_type";
+ private static final String FIELD_NAME_IN_ORIDNAL_POSITION_PATH = "/field_name_in_ordinal_position";
+ private static final String FIELD_TYPE_IN_ORIDNAL_POSITION_PATH = "/field_type_in_ordinal_position";
+ private static final String RECORD_PATH = "/record";
+ private static final String METRICS_PATH = "/metrics";
+ private static final String CPU_PATH = "/cpu";
+ private static final String ESTIMATOR_PATH = "/estimator";
+ private static final String CONFUSION_MATRIX_PATH = "/confusion_matrix";
+ private static final String LIVENESS_PATH = "/liveness";
+ private static final String TABLE_INFO_PATH = "/table_info";
+ private static final String TRAIN_TASK_NAME_BY_ESTIMATOR_ID_PATH = "/train_task_name_by_estimator_id";
+ private static final String TRAIN_TASK_CSV_CONTENT_PATH = "/train_task_csv_content";
+ private static final String METRICS_CSV_CONTENT_PATH = "/metrics_csv_content";
+ private static final String COLUMNS_CONTAIN_ONLY_DISTINCT_VALUES_PATH = "/columns_contain_only_distinct_values";
+ private static final String REDUCED_AND_EVAULATION_PATH = "/reduced_and_evaulation";
+ private static final String MAX_RETROSPECTIVE_DAY = "/max_retrospective_day";
+
+ @Value("${home.directory}") private String homeDirectory;
+ @Value("${evaluation.csv.directory}") private String evaluationCsvDirectory;
+ @Value("${trainUrl}") String trainUrl;
+ @Autowired private RestTemplate restTemplate;
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ public String getHomeDirectory() {
+ return homeDirectory;
+ }
+ public String getEvaluationCsvDirectory() {
+ return evaluationCsvDirectory;
+ }
+
+ public TrainModuleStatus testLiveness() {
+ TrainModuleStatus trainModuleStatus = null;
+ String url = trainUrl + LIVENESS_PATH;
+ try {
+ LivenessDto livenessDto = restTemplate.getForEntity(url, LivenessDto.class).getBody();
+ if (livenessDto.getResponse().equals(OK)) {
+ trainModuleStatus = TrainModuleStatus.OK;
+ }
+ } catch (ResourceAccessException exception) {
+ if (exception.getCause() instanceof ConnectException) {
+ trainModuleStatus = TrainModuleStatus.NOT_ACCESSIBLE;
+ }
+ if (exception.getCause() instanceof SocketTimeoutException) {
+ //TO-DO logging
+ trainModuleStatus = TrainModuleStatus.RESPONSE_TIMEOUT;
+ }
+ }
+ return trainModuleStatus;
+ }
+
+ public int getCpuCoreNumber() {
+ String url = trainUrl + CPU_PATH;
+ int coreNumber = restTemplate.getForEntity(url, Integer.class).getBody();
+ return coreNumber;
+ }
+
+ public String[] getSchemas() {
+ String url = trainUrl + SCHEMA_PATH;
+ String[] schemas = restTemplate.getForEntity(url, String[].class).getBody();
+ return schemas;
+ }
+
+ public String[] getTables(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TABLE_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public TableProperties getTableProperties(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TABLE_INFO_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<TableProperties> tableProperties =
+ restTemplate.getForEntity(urlTemplate, TableProperties.class, params);
+ TableProperties result = tableProperties.getBody();
+ return result;
+ }
+
+ public String[] getFieldNameInOrdinalPosition(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_NAME_IN_ORIDNAL_POSITION_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public String[] getFieldTypeInOrdinalPosition(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_TYPE_IN_ORIDNAL_POSITION_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ String[] tables = restTemplate.getForEntity(urlTemplate, String[].class, params).getBody();
+ return tables;
+ }
+
+ public Map<String, String> getFieldTypeByName(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + FIELD_NAME_AND_TYPE_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<Map<String, String>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ Map<String, String> result =
+ restTemplate.exchange(urlTemplate, HttpMethod.GET, null, typeReference, params).getBody();
+ return result;
+ }
+
+ public String getRecordsAsString(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + RECORD_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+ public List<List<Object>> getRecords(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + RECORD_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<List<List<Object>>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ List<List<Object>> records = exchangeAsList(urlTemplate, typeReference, params);
+ return records;
+ }
+
+ public List<EncodingAndFeatureEngineeringParameters> getEncodingAndFeatureEngineeringParameters(
+ Map<String, String> queryParamValueByName) {
+ String url = trainUrl + ENCODING_AND_FEATURE_ENGINEERING_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<List<EncodingAndFeatureEngineeringParameters>> typeReference =
+ new ParameterizedTypeReference<>() {
+ };
+ List<EncodingAndFeatureEngineeringParameters> encodingAndFeatureEngineeringParameters =
+ exchangeAsList(urlTemplate, typeReference, params);
+ return encodingAndFeatureEngineeringParameters;
+ }
+
+ public Map<Integer, TrainTaskParameters> getTrainTask(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TRAIN_TASK_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<Map<Integer, TrainTaskParameters>> typeReference =
+ new ParameterizedTypeReference<>() {
+ };
+ Map<Integer, TrainTaskParameters> result =
+ restTemplate.exchange(urlTemplate, HttpMethod.GET, null, typeReference, params).getBody();
+ return result;
+ }
+
+ public Map<String, String> getFeatureSelectors() {
+ return useCommmonQuery("/available_feature_selector");
+ }
+
+ public Map<String, String> getSamplers() {
+ return useCommmonQuery("/available_sampler");
+ }
+
+ public Map<String, String> getScalers() {
+ return useCommmonQuery("/available_scaler");
+ }
+
+ public Map<String, String> getModels() {
+ return useCommmonQuery("/available_model");
+ }
+
+ public Map<String, String> useCommmonQuery(String path) {
+ String url = trainUrl + path;
+ Map<String, String> parameters = restTemplate.getForEntity(url, Map.class).getBody();
+ return parameters;
+ }
+
+ public List<EstimatorAndTrainTaskIdentifiers> getEstimatorIdentifiers() {
+ String url = trainUrl + "/estimator";
+ ParameterizedTypeReference<List<EstimatorAndTrainTaskIdentifiers>> typeReference =
+ new ParameterizedTypeReference<>() {
+ };
+ List<EstimatorAndTrainTaskIdentifiers> estimatorAndTrainTaskIdentifiers =
+ exchangeAsList(url, typeReference, Collections.emptyMap());
+ return estimatorAndTrainTaskIdentifiers;
+ }
+
+ public List<Map<String, Double>> getMetrics(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + METRICS_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ParameterizedTypeReference<List<Map<String, Double>>> typeReference = new ParameterizedTypeReference<>() {
+ };
+ List<Map<String, Double>> metrics = exchangeAsList(urlTemplate, typeReference, params);
+ return metrics;
+ }
+
+ public ConfusionMatrixDto getConfusionMatrixElements(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + CONFUSION_MATRIX_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<ConfusionMatrixDto> responseEntity =
+ restTemplate.getForEntity(urlTemplate, ConfusionMatrixDto.class, params);
+ return responseEntity.getBody();
+ }
+
+ public String getTrainTaskNameByEstimatorId(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + TRAIN_TASK_NAME_BY_ESTIMATOR_ID_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ ResponseEntity<String> tableProperties = restTemplate.getForEntity(urlTemplate, String.class, params);
+ String result = tableProperties.getBody();
+ return result;
+ }
+
+ public CsvContentDto getTrainTaskCsvContent() {
+ String url = trainUrl + TRAIN_TASK_CSV_CONTENT_PATH;
+ return restTemplate.getForEntity(url, CsvContentDto.class, Collections.emptyMap()).getBody();
+ }
+
+ public CsvContentDto getMetricsCsvContent() {
+ String url = trainUrl + METRICS_CSV_CONTENT_PATH;
+ return restTemplate.getForEntity(url, CsvContentDto.class, Collections.emptyMap()).getBody();
+ }
+
+ public List<String> getColumnsContainOnlyDistinctValues(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + COLUMNS_CONTAIN_ONLY_DISTINCT_VALUES_PATH;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ return Arrays.stream(restTemplate.getForEntity(urlTemplate, String[].class, params).getBody())
+ .collect(Collectors.toList());
+ }
+
+ public int getMaxRetroSpectiveDay(Map<String, String> queryParamValueByName) {
+ String url = trainUrl + MAX_RETROSPECTIVE_DAY;
+ UrlBuilder urlBuilder = new UrlBuilder();
+ urlBuilder.build(url, queryParamValueByName);
+ String urlTemplate = urlBuilder.getUrl();
+ Map<String, String> params = urlBuilder.getQueryParams();
+ return restTemplate.getForEntity(urlTemplate, Integer.class, params).getBody();
+ }
+
+ public int addEncodingAndFeatureEngineeringParameters(EncodingAndFeatureEngineeringParameters parameters) {
+ return addGenericParameters(parameters, ENCODING_AND_FEATURE_ENGINEERING_PATH);
+ }
+
+ public int addTrainTask(TrainTaskParameters trainTaskParameters) {
+ return addGenericParameters(trainTaskParameters, TRAIN_TASK_PATH);
+ }
+
+ public ReducingAndEvaulationParameterResponseDto createReducedAndEvaulationTables(ReducingAndEvaulationParameterRequestDto requestDto) {
+ String url = trainUrl + REDUCED_AND_EVAULATION_PATH;
+ return restTemplate.postForEntity(url, requestDto, ReducingAndEvaulationParameterResponseDto.class, Collections.emptyMap()).getBody();
+ }
+
+ public void fit(FitParameters parameters) {
+ addGenericParameters(parameters, FIT_PATH);
+ }
+
+ public <T> Integer addGenericParameters(T parameters, String path) {
+ String url = trainUrl + path;
+ try {
+ ResponseEntity<Integer> responseFromTrainModule =
+ restTemplate.postForEntity(url, parameters, Integer.class);
+ return responseFromTrainModule.getBody();
+ } catch (HttpClientErrorException httpClientErrorException) {
+ String errorMessage = httpClientErrorException.getResponseBodyAsString();
+ log.error(errorMessage);
+ throw new RuntimeException(errorMessage);
+ }
+ }
+
+ private <T> List<T> exchangeAsList(String uri, ParameterizedTypeReference<List<T>> responseType,
+ Map<String, String> uriVariables) {
+ return restTemplate.exchange(uri, HttpMethod.GET, null, responseType, uriVariables).getBody();
+ }
+
+}
+
--- /dev/null
+package hu.user.trainmanager.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.web.util.UriComponentsBuilder;
+
+public class UrlBuilder {
+
+ private String url;
+ private Map<String, String> queryParams = new HashMap<>();
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Map<String, String> getQueryParams() {
+ return queryParams;
+ }
+
+ public void build(String url, Map<String, String> queryValueByParamName) {
+ List<String> paramNames = new ArrayList<>();
+ List<String> paramValues = new ArrayList<>();
+ for (var entry : queryValueByParamName.entrySet()) {
+ paramNames.add(entry.getKey());
+ paramValues.add(entry.getValue());
+ }
+ buildUrl(url, paramNames);
+ fillQueryParams(paramNames, paramValues);
+ }
+
+ private void buildUrl(String url, List<String> queryParamNames) {
+ String urlTemplate;
+ UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromHttpUrl(url);
+ for (String queryParamName : queryParamNames) {
+ uriComponentsBuilder = uriComponentsBuilder.queryParam(queryParamName, "{" + queryParamName + "}");
+ }
+ this.url = uriComponentsBuilder.encode().toUriString();
+ }
+
+ private void fillQueryParams(List<String> paramNames, List<String> paramValuee) {
+ Map<String, String> params = new HashMap<>();
+ for (int i = 0; i < paramNames.size(); i++) {
+ params.put(paramNames.get(i), paramValuee.get(i));
+ }
+ this.queryParams = params;
+ }
+}
--- /dev/null
+package hu.user.trainmanager.status;
+
+public enum TrainModuleStatus {
+ OK,
+ NOT_ACCESSIBLE,
+ RESPONSE_TIMEOUT
+}
--- /dev/null
+package hu.user.trainmanager.status;
+
+public enum TrainStatus {
+ OK,
+ NOT_ACCESSIBLE,
+ RESPONSE_TIMEOUT
+}
--- /dev/null
+package hu.user.trainmanager.status;
+
+public enum TrainStepName {
+
+ ENCODING,
+ FEATURE_ENGINEERING,
+ SAMPLING,
+ FITTING,
+ ESTIMATOR_PERSISTING,
+ METRIC_PERSISTING
+}
--- /dev/null
+package hu.user.trainmanager.status;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class TrainStepState {
+
+ private int trainTaskId;
+ private TrainStepName trainStepName;
+ private TrainStepStatus trainStepStatus;
+
+}
+
--- /dev/null
+package hu.user.trainmanager.status;
+
+public enum TrainStepStatus {
+ READY, PREVIOUSLY_DONE, STARTED, FINISHED
+}
--- /dev/null
+package hu.user.trainmanager.status;
+
+public enum TrainTaskState {
+
+ STORED,
+ IN_PROGRESS,
+ DONE
+}
--- /dev/null
+package hu.user.trainmanager.viewmodel;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.zkoss.bind.annotation.Command;
+import org.zkoss.bind.annotation.Init;
+import org.zkoss.bind.annotation.NotifyChange;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.ListModelList;
+import org.zkoss.zul.Messagebox;
+
+import antlr.collections.impl.IntRange;
+import hu.user.trainmanager.dto.FraudCandidate;
+import hu.user.trainmanager.dto.TableProperties;
+import hu.user.trainmanager.dto.csv.ReducingAndEvaulationParameterRequestDto;
+import hu.user.trainmanager.dto.csv.ReducingAndEvaulationParameterResponseDto;
+import hu.user.trainmanager.service.CsvHandler;
+import hu.user.trainmanager.service.TrainClient;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+@Getter
+@Setter
+@Slf4j
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class DatabaseViewModel {
+
+ @WireVariable private TrainClient trainClient;
+
+ @WireVariable private CsvHandler csvHandler;
+
+ private static final String SCHEMA_NAME = "schema_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String LIMIT = "limit";
+ private static final String OFFSET = "offset";
+ private static final int DEFAULT_SLIDER_MAX_VALUE = 10;
+
+ private String schema;
+ private List<String> schemas = new ArrayList<>();
+ private String table;
+ private List<String> tables = new ArrayList<>();
+ private TableProperties tableProperties;
+ private Integer recordNumber;
+ private ListModelList<Integer> offsetValues = new ListModelList<>();
+ private Integer offset;
+ private ListModelList<Integer> limitValues = new ListModelList<>();
+ private Integer limit;
+ private Map<String, String> fieldTypeByName = new HashMap<>();
+ private List<String> fieldNames = new ArrayList<>();
+ private List<String> fieldTypes = new ArrayList<>();
+
+ private ListModelList<List<Object>> records = new ListModelList<>();
+ private String actualFooter;
+ private List<String> fraudTypeFieldNames = new ArrayList<>();
+ private String fraudTypeFieldName;
+ private int desiredPositiveFraudNumber;
+ private int desiredNegativeFraudNumber;
+ private boolean enableAddDateTimeSuffix;
+ private boolean showTypeChecked;
+ private boolean showSplitChecked;
+ private Integer positiveCaseNumber = DEFAULT_SLIDER_MAX_VALUE;
+ private Integer negativeCaseNumber = DEFAULT_SLIDER_MAX_VALUE;
+
+ @Init
+ public void init() {
+ schemas = Arrays.stream(trainClient.getSchemas()).collect(Collectors.toList());
+// ReducedAndEvaulationParameterRequestDto requestDto =
+// new ReducedAndEvaulationParameterRequestDto("card_10000_5", "transaction", "fraud", 5, 6,true);
+// Map<Integer, Integer> result = trainClient.createReducedAndEvaulationTables(requestDto);
+// System.out.println(result);
+ }
+
+ @Command
+ @NotifyChange("tables")
+ public void changeSchema() {
+ if (schema != null && schemas.contains(schema)) {
+ tables = List.of(trainClient.getTables(Map.of(SCHEMA_NAME, schema)));
+ }
+ }
+
+ @Command
+ @NotifyChange("tables")
+ public void refreshTables() {
+ if (schema != null) {
+ tables = List.of(trainClient.getTables(Map.of(SCHEMA_NAME, schema)));
+ }
+ }
+
+ @Command
+ @NotifyChange({"records", "recordNumber", "offsetValues", "limitValues", "fieldNames", "fieldTypes"})
+ public void changeTable() {
+ if (schema != null && table != null) {
+ limit = null;
+ offset = null;
+ records.clear();
+ limitValues.clear();
+ offsetValues.clear();
+ tableProperties = trainClient.getTableProperties(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ List<FraudCandidate> fraudCandidates = tableProperties.getFraudCandidates();
+ fraudTypeFieldNames = fraudCandidates.stream().map(e -> e.getName()).collect(Collectors.toList());
+ recordNumber = tableProperties.getRecordNumber();
+ if (recordNumber <= 100) {
+ offsetValues = new ListModelList<>(List.of(0));
+ limitValues =
+ new ListModelList<>(IntStream.range(1, recordNumber).boxed().collect(Collectors.toList()));
+ } else {
+ int groupNumber = recordNumber / 100;
+ for (int i = 0; i < groupNumber; i++) {
+ offsetValues.add(i * 100);
+ }
+ limitValues = new ListModelList<>(IntStream.range(1, 100).boxed().collect(Collectors.toList()));
+ }
+ fieldTypeByName = trainClient.getFieldTypeByName(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ fieldNames = Arrays.stream(
+ trainClient.getFieldNameInOrdinalPosition(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table)))
+ .collect(Collectors.toList());
+ fieldTypes = Arrays.stream(
+ trainClient.getFieldTypeInOrdinalPosition(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table)))
+ .collect(Collectors.toList());
+ if (limit != null && offset != null) {
+ records = fillRecords();
+ }
+ }
+ }
+
+ @Command
+ @NotifyChange("records")
+ public void changeOffset() {
+ if (limit != null && offset != null) {
+ records = fillRecords();
+ }
+ }
+
+ @Command
+ @NotifyChange("records")
+ public void changeLimit() {
+ if (limit != null && offset != null) {
+ records = fillRecords();
+ }
+ }
+
+ @Command
+ @NotifyChange({"positiveCaseNumber", "negativeCaseNumber"})
+ public void refreshFraudCaseNumbers() {
+ if (fraudTypeFieldName != null && schema != null && table != null) {
+ tableProperties = trainClient.getTableProperties(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ List<FraudCandidate> fraudCandidates = tableProperties.getFraudCandidates();
+ for (FraudCandidate fraudCandidate : fraudCandidates) {
+ if (fraudCandidate.getName().equals(fraudTypeFieldName)) {
+ positiveCaseNumber = fraudCandidate.getFraudNumber();
+ negativeCaseNumber = fraudCandidate.getNoFraudNumber();
+ }
+ log.info("Positive fraud case number: {}", positiveCaseNumber);
+ log.info("Negative fraud case number: {}", negativeCaseNumber);
+ }
+ }
+ }
+
+ @Command
+ public void splitDatabase() {
+ List<FraudCandidate> fraudCandidates = tableProperties.getFraudCandidates();
+ int positiveFraudNumber = 0;
+ int negativeFraudNumber = 0;
+ for (FraudCandidate fraudCandidate : fraudCandidates)
+ if (fraudCandidate.getName().equals(fraudTypeFieldName)) {
+ if (desiredPositiveFraudNumber < fraudCandidate.getFraudNumber()) {
+ positiveFraudNumber = desiredPositiveFraudNumber;
+ } else {
+ positiveFraudNumber = fraudCandidate.getFraudNumber();
+ }
+ if (desiredNegativeFraudNumber < fraudCandidate.getNoFraudNumber()) {
+ negativeFraudNumber = desiredNegativeFraudNumber;
+ } else {
+
+ negativeFraudNumber = fraudCandidate.getNoFraudNumber();
+ }
+ }
+ ReducingAndEvaulationParameterRequestDto requestDto =
+ new ReducingAndEvaulationParameterRequestDto(schema, table, fraudTypeFieldName, positiveFraudNumber,
+ negativeFraudNumber, enableAddDateTimeSuffix);
+ ReducingAndEvaulationParameterResponseDto responseFromTrainModule =
+ trainClient.createReducedAndEvaulationTables(requestDto);
+ String csvPath = trainClient.getEvaluationCsvDirectory();
+ String plannedCsvFileName = responseFromTrainModule.getPlannedCsvFileName();
+ Map<Integer, Integer> plannedCsvContent = responseFromTrainModule.getPlannedCsvContent();
+ csvHandler.createEvaulationCsvFile(csvPath, plannedCsvFileName, plannedCsvContent);
+ Messagebox.show("The table has been split and csv file created.");
+ }
+
+ private ListModelList<List<Object>> fillRecords() {
+ return new ListModelList<>(trainClient.getRecords(
+ Map.of(SCHEMA_NAME, schema, TABLE_NAME, table, LIMIT, String.valueOf(limit), OFFSET,
+ String.valueOf(offset))));
+ }
+}
--- /dev/null
+package hu.user.trainmanager.viewmodel;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.zkoss.bind.BindUtils;
+import org.zkoss.bind.ValidationContext;
+import org.zkoss.bind.Validator;
+import org.zkoss.bind.annotation.BindingParam;
+import org.zkoss.bind.annotation.Command;
+import org.zkoss.bind.annotation.ContextParam;
+import org.zkoss.bind.annotation.ContextType;
+import org.zkoss.bind.annotation.Init;
+import org.zkoss.bind.annotation.NotifyChange;
+import org.zkoss.bind.validator.AbstractValidator;
+import org.zkoss.zk.ui.Component;
+import org.zkoss.zk.ui.Desktop;
+import org.zkoss.zk.ui.Executions;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.ListModelList;
+import org.zkoss.zul.Messagebox;
+
+import hu.user.trainmanager.dto.EncodingAndFeatureEngineeringParameters;
+import hu.user.trainmanager.dto.FeatureEnginneringParameters;
+import hu.user.trainmanager.dto.FieldProperty;
+import hu.user.trainmanager.dto.FraudCandidate;
+import hu.user.trainmanager.dto.TableProperties;
+import hu.user.trainmanager.dto.TrainTaskParameters;
+import hu.user.trainmanager.dto.display.FieldNameAndEncodingType;
+import hu.user.trainmanager.service.TrainClient;
+import hu.user.trainmanager.status.TrainTaskState;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class InputTrainParameterViewModel {
+
+ private static final String GET_CPU = "get_cpu";
+ private static final String GET_SCHEMAS = "get_schemas";
+ private static final String GET_TABLES = "get_tables";
+ private static final String SCHEMA_NAME = "schema_name";
+ private static final String TABLE_NAME = "table_name";
+ private static final String TIME_BASE_FIELD_NAME = "time_base_field_name";
+ private static final String GET_TABLE_PROPERTY = "get_table_property";
+ private static final String GET_RECORDS = "get_records";
+ private static final String LIMIT = "limit";
+ private static final String OFFSET = "offset";
+ private static final String POST_E_AND_FE_PARAMETERS = "new_e_and_fe_parameters";
+ private static final String GET_E_AND_FE_PARAMETERS = "get_e_and_fe_parameters";
+ private static final String GET_FEATURE_SELECTOR = "get_feature_selector";
+ private static final String GET_SAMPLER = "get_sampler";
+ private static final String GET_SCALER = "get_scaler";
+ private static final String GET_MODEL = "get_model";
+ private static final String NEW_TRAIN_TASK = "new_train_task";
+ private static final String GET_TRAIN_TASK = "get_train_task";
+ private static final String FIT = "fit";
+ private static final String GET_METRICS = "get_metrics";
+ private static final String GET_ESTIMATOR = "get_estimator";
+ private static final String ESTIMATOR_ID = "estimator_id";
+ private static final String ID = "id";
+ private static final String INT = "int";
+ private static final String FLOAT = "float";
+ private static final String DOUBLE = "double";
+ private static final String VARCHAR = "varchar";
+ private static final String DATE_TIME = "datetime";
+ private static final String JULIAN = "julian";
+ private static final String LABEL_ENCODER = "label_encoder";
+ private static final String FRAUD_TYPE = "fraud_type";
+ private static final String OMIT = "omit";
+ private static final String CARD_NUMBER = "card_number";
+
+ private static final List<String> APPLICABLE_TYPES_FOR_FE = List.of(INT, FLOAT, DOUBLE);
+
+ @WireVariable private TrainClient trainClient;
+
+ private String schema;
+ private String table;
+ private String featureSelector;
+ private String sampler;
+ private String scaler;
+ private String model;
+
+ private int cpuCore;
+
+ private List<String> schemas = new ArrayList<>();
+ private List<String> tables = new ArrayList<>();
+ private TableProperties tableProperties;
+ private Map<String, String> encodingParameters = new HashMap<>();
+
+ private boolean enabledFeatureEngineering;
+ private FeatureEnginneringParameters featureEnginneringParameters;
+
+ private List<String> eligibleTimeBaseFields;
+ private String chosenTimeBaseField;
+ private List<String> fieldCandidatesForFeatureEngineering = new ArrayList<>();
+ private String chosenFeatureField;
+ private List<String> fields = new ArrayList<>();
+ private int fieldIndex;
+
+ private String currentField;
+
+ private String currentEncodingType;
+ private ListModelList<String> currentApplicableEncodingTypes = new ListModelList<>();
+
+ private List<String> availableEncodingTypes = new ArrayList<>();
+
+ private boolean disabledNextEncodingTypeButton;
+ private boolean disabledFinishEncodingTypeButton;
+ private Integer encodingAndFeatureEngineeringId;
+ private Map<String, String> featureSelectorById = new HashMap<>();
+ private String featureSelectorName;
+ private Map<String, String> samplerById = new HashMap<>();
+
+ private String samplerName;
+ private Map<String, String> scalerById = new HashMap<>();
+ private String scalerName;
+ private Map<String, String> modelById = new HashMap<>();
+ private String modelName;
+ private Integer testSizePercent;
+ private Integer retroSpectiveDay;
+ private Integer maxRetroSpectiveDay;
+
+ private List<Integer> intervals = new ArrayList<>();
+ private String presentedIntervals;
+
+ private List<EncodingAndFeatureEngineeringParameters> storedEncodingAndFeatureEngineeringParameters;
+
+ private EncodingAndFeatureEngineeringParameters selectedStoredEncodingAndFeatureEngineeringParameter;
+
+ private List<FieldNameAndEncodingType> displayedEncodingParameters;
+
+ private String displayedChosenFeatureField;
+ private String displayedRetroSpectiveIntervals;
+
+ private String trainTaskName;
+ private boolean allEncodingParametersSet;
+ private Validator trainTaskNameValidator = getTrainTaskNameValidator();
+ private List<String> columnsContainOnlyDistinctValues = new ArrayList<>();
+ private String uniqueFieldName;
+ private String encodingErrorMessage;
+ private String featureEngineeringErrorMessage;
+
+ @Init
+ public void init(@ContextParam(ContextType.COMPONENT) Component component) {
+ schemas = List.of(trainClient.getSchemas());
+ featureSelectorById = trainClient.getFeatureSelectors();
+ samplerById = trainClient.getSamplers();
+ scalerById = trainClient.getScalers();
+ modelById = trainClient.getModels();
+ disabledNextEncodingTypeButton = true;
+ disabledFinishEncodingTypeButton = true;
+ }
+
+ private void clearPipeLineParameters() {
+ featureSelectorName = null;
+ samplerName = null;
+ scalerName = null;
+ modelName = null;
+ testSizePercent = null;
+ trainTaskName = null;
+ BindUtils.postNotifyChange(null, null, this, "featureSelectorName");
+ BindUtils.postNotifyChange(null, null, this, "samplerName");
+ BindUtils.postNotifyChange(null, null, this, "scalerName");
+ BindUtils.postNotifyChange(null, null, this, "modelName");
+ BindUtils.postNotifyChange(null, null, this, "testSizePercent");
+ BindUtils.postNotifyChange(null, null, this, "trainTaskName");
+
+ }
+
+ public ListModelList<String> getfeatureSelectors() {
+ return new ListModelList<>(getListFromMap(featureSelectorById));
+ }
+
+ public List<String> getSamplers() {
+ return getListFromMap(samplerById);
+ }
+
+ public List<String> getScalers() {
+ return getListFromMap(scalerById);
+ }
+
+ public List<String> getModels() {
+ return getListFromMap(modelById);
+ }
+
+ public List<String> getListFromMap(Map<String, String> input) {
+ return input.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
+ }
+
+ private String getKeyFromMapByValue(Map<String, String> map, String value) {
+ String result =
+ map.entrySet().stream().filter(e -> e.getValue().equals(value)).map(e -> e.getKey()).findFirst().get();
+ return result;
+ }
+
+ @Command
+ @NotifyChange("tables")
+ public void changeSchema() {
+ if (schema != null && schemas.contains(schema)) {
+ tables = List.of(trainClient.getTables(Map.of(SCHEMA_NAME, schema)));
+ }
+ }
+
+ @Command
+ @NotifyChange({"fieldCandidatesForFeatureEngineering", "fields", "fieldIndex", "fieldIndex", "currentField",
+ "disabledNextEncodingTypeButton", "disabledFinishEncodingTypeButton", "eligibleTimeBaseFields"})
+ public void changeTable() {
+ if (schema != null && table != null) {
+ tableProperties = trainClient.getTableProperties(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ columnsContainOnlyDistinctValues =
+ trainClient.getColumnsContainOnlyDistinctValues(Map.of(SCHEMA_NAME, schema, TABLE_NAME, table));
+ encodingParameters = new HashMap<>();
+ collectEligibleTimeBaseFields();
+ collectFieldCandidatesForFeatureEngineering();
+ collectFieldNames();
+ }
+ }
+
+ @Command
+ public void changeUniqueField() {
+ if (chosenTimeBaseField != null) {
+ initBeforeChooseEncodingTypes();
+ }
+ }
+
+ @Command
+ public void changeTimeBaseField() {
+ if (uniqueFieldName != null) {
+ initBeforeChooseEncodingTypes();
+ }
+ maxRetroSpectiveDay = trainClient.getMaxRetroSpectiveDay(
+ Map.of(SCHEMA_NAME, schema, TABLE_NAME, table, TIME_BASE_FIELD_NAME, chosenTimeBaseField));
+ }
+
+ private void initBeforeChooseEncodingTypes() {
+ fieldIndex = 0;
+ setCurrentFieldValue(0);
+ disabledNextEncodingTypeButton = false;
+ disabledFinishEncodingTypeButton = true;
+ BindUtils.postNotifyChange(null, null, this, "disabledNextEncodingTypeButton");
+ BindUtils.postNotifyChange(null, null, this, "disabledFinishEncodingTypeButton");
+ }
+
+ private void collectEligibleTimeBaseFields() {
+ eligibleTimeBaseFields =
+ tableProperties.getFields().stream().filter(e -> e.getType().equals(DATE_TIME)).map(e -> e.getName())
+ .collect(Collectors.toList());
+ }
+
+ private void collectFieldCandidatesForFeatureEngineering() {
+ if (tableProperties != null) {
+ List<FieldProperty> fieldProperties = tableProperties.getFields();
+ fieldCandidatesForFeatureEngineering =
+ fieldProperties.stream().filter(e -> APPLICABLE_TYPES_FOR_FE.contains(e.getType()))
+ .map(e -> e.getName()).collect(Collectors.toList());
+ }
+ }
+
+ private void collectFieldNames() {
+ if (tableProperties != null) {
+ fields = tableProperties.getFields().stream().map(e -> e.getName()).collect(Collectors.toList());
+ if (tableProperties.getPrimaryKey() != null) {
+ fields.remove(tableProperties.getPrimaryKey());
+ }
+ }
+ }
+
+ private void setCurrentFieldValue(int fieldIndex) {
+ currentField = fields.get(fieldIndex);
+ BindUtils.postNotifyChange(null, null, this, "currentField");
+ setCurrentApplicableEncodingTypes();
+ }
+
+ public void setCurrentApplicableEncodingTypes() {
+ Optional<FieldProperty> optionalFieldProperty =
+ tableProperties.getFields().stream().filter(e -> e.getName().equals(currentField)).findFirst();
+ if (optionalFieldProperty.isPresent()) {
+ FieldProperty fieldProperty = optionalFieldProperty.get();
+ String type = fieldProperty.getType();
+ List<String> otherApplicableEncodingTypes = List.of(fieldProperty.getOtherApplicableEncodingTypes());
+ currentApplicableEncodingTypes.add(OMIT);
+ switch (type) {
+ case DATE_TIME:
+ currentApplicableEncodingTypes.add(JULIAN);
+ break;
+ case VARCHAR:
+ currentApplicableEncodingTypes.add(LABEL_ENCODER);
+ currentApplicableEncodingTypes.addAll(otherApplicableEncodingTypes);
+ break;
+ case INT:
+ currentApplicableEncodingTypes.add(INT);
+ List<FraudCandidate> fraudCandidates = tableProperties.getFraudCandidates();
+ List<String> fraudCandidateFieldNames =
+ fraudCandidates.stream().map(e -> e.getName()).collect(Collectors.toList());
+ if (fraudCandidateFieldNames.contains(fieldProperty.getName())) {
+ currentApplicableEncodingTypes.add(FRAUD_TYPE);
+ }
+ break;
+ case FLOAT:
+ currentApplicableEncodingTypes.add(FLOAT);
+ break;
+ }
+ }
+ }
+
+ @Command
+ @NotifyChange({"disabledNextEncodingTypeButton", "disabledFinishEncodingTypeButton",
+ "currentApplicableEncodingTypes", "currentEncodingType", "currentField"})
+ public void saveCurrentFieldAndEncodingTypeAndGetNextField() {
+ if (currentEncodingType != null && !currentEncodingType.isEmpty()) {
+ saveCurrentFieldAndEncodingType();
+ fieldIndex++;
+ }
+ if (fieldIndex == fields.size() - 1) {
+ disabledNextEncodingTypeButton = true;
+ disabledFinishEncodingTypeButton = false;
+ }
+ currentApplicableEncodingTypes.clear();
+ currentEncodingType = null;
+ currentField = null;
+// BindUtils.postNotifyChange(null, null, this, "currentApplicableEncodingTypes");
+// BindUtils.postNotifyChange(null, null, this, "currentEncodingType");
+ if (fieldIndex <= fields.size() - 1) {
+ setCurrentFieldValue(fieldIndex);
+ }
+ }
+
+ @Command
+ @NotifyChange({"disabledFinishEncodingTypeButton", "currentApplicableEncodingTypes", "currentEncodingType",
+ "currentField", "encodingErrorMessage"})
+ public void saveLastFieldAndEncodingType() {
+ if (currentEncodingType != null && !currentEncodingType.isEmpty()) {
+ saveCurrentFieldAndEncodingType();
+ disabledFinishEncodingTypeButton = true;
+ currentApplicableEncodingTypes.clear();
+ currentEncodingType = null;
+ currentField = null;
+
+ if (!encodingParameters.values().contains(FRAUD_TYPE)) {
+ encodingErrorMessage = "Fraud type field is missing";
+ }
+ if (encodingParameters.get(chosenTimeBaseField).equals(OMIT)) {
+ encodingErrorMessage = "Time base field has been omitted";
+ }
+ if (encodingParameters.keySet().contains(CARD_NUMBER) && encodingParameters.get(CARD_NUMBER).equals(OMIT)) {
+ encodingParameters.put(CARD_NUMBER, LABEL_ENCODER);
+ }
+ System.out.println();
+ }
+ }
+
+ public void saveCurrentFieldAndEncodingType() {
+ encodingParameters.put(currentField, currentEncodingType);
+ if (encodingParameters.size() == fields.size()) {
+ allEncodingParametersSet = true;
+ }
+ }
+
+ @Command
+ @NotifyChange("encodingErrorMessage")
+ public void clearEncodingParametersAfterError() {
+ encodingErrorMessage = null;
+ clearEncodingParameters();
+ if (schema != null && table != null && uniqueFieldName != null && chosenTimeBaseField != null) {
+ setCurrentFieldValue(0);
+ fieldIndex = 0;
+ }
+ disabledNextEncodingTypeButton = false;
+ BindUtils.postNotifyChange(null, null, this, "disabledNextEncodingTypeButton");
+ }
+
+ @Command
+ @NotifyChange("presentedIntervals")
+ public void addDayToIntervals() {
+ if (!intervals.contains(retroSpectiveDay) && retroSpectiveDay < maxRetroSpectiveDay) {
+ intervals.add(retroSpectiveDay);
+ presentedIntervals = intervals.stream().map(e -> String.valueOf(e)).collect(Collectors.joining("; "));
+ }
+ }
+
+ @Command
+ @NotifyChange("featureEngineeringErrorMessage")
+ public void checkIfSelectedFeatureFieldExist() {
+ if (encodingParameters.keySet().contains(chosenFeatureField) && encodingParameters.get(chosenFeatureField)
+ .equals(OMIT)) {
+ featureEngineeringErrorMessage = "The field has been omitted";
+ } else {
+ featureEngineeringErrorMessage = null;
+ }
+ }
+
+ public void saveEncodingAndFeatureEngineering() {
+ int[] intervalsAsArray = new int[intervals.size()];
+ for (int i = 0; i < intervals.size(); i++) {
+ intervalsAsArray[i] = intervals.get(i);
+ }
+ filterEncodingParameters();
+ FeatureEnginneringParameters feParameters = null;
+ if (enabledFeatureEngineering && !chosenFeatureField.isEmpty() && intervalsAsArray.length > 0) {
+ feParameters = new FeatureEnginneringParameters(chosenFeatureField, intervalsAsArray);
+ }
+ EncodingAndFeatureEngineeringParameters efeParameters =
+ new EncodingAndFeatureEngineeringParameters(schema, table, chosenTimeBaseField, encodingParameters,
+ feParameters);
+ encodingAndFeatureEngineeringId = trainClient.addEncodingAndFeatureEngineeringParameters(efeParameters);
+ }
+
+ @Command
+ public void saveTrainTask() {
+ saveEncodingAndFeatureEngineering();
+ int featureSelectorCode = Integer.parseInt(getKeyFromMapByValue(featureSelectorById, featureSelectorName));
+ int samplerCode = Integer.parseInt(getKeyFromMapByValue(samplerById, samplerName));
+ int scalerCode = Integer.parseInt(getKeyFromMapByValue(scalerById, scalerName));
+ int modelCode = Integer.parseInt(getKeyFromMapByValue(modelById, modelName));
+ double testSize = testSizePercent * 0.01;
+ TrainTaskParameters trainTaskParameters =
+ new TrainTaskParameters(trainTaskName, uniqueFieldName, encodingAndFeatureEngineeringId,
+ featureSelectorCode, samplerCode, scalerCode, modelCode, testSize, TrainTaskState.STORED);
+ int trainTaskId = trainClient.addTrainTask(trainTaskParameters);
+ Messagebox.show("Train Task persisted, train task id: " + trainTaskId);
+ clearInputParametersOnForm();
+ BindUtils.postGlobalCommand(null, null, "automaticRefreshTrainTasks", null);
+ }
+
+ private void clearInputParametersOnForm() {
+ clearEncodingParameters();
+ clearFeatureEngineeringParameters();
+ clearPipeLineParameters();
+ if (schema != null && table != null && uniqueFieldName != null && chosenTimeBaseField != null) {
+ initBeforeChooseEncodingTypes();
+ }
+
+ }
+
+ @Command
+ @NotifyChange("allEncodingParametersSet")
+ public void clearAllEncodingParameters() {
+ clearDatabaseParameters();
+ clearSpecificProperties();
+ clearEncodingParameters();
+ clearFeatureEngineeringParameters();
+ clearPipeLineParameters();
+ fieldIndex = 0;
+ }
+
+ @Command
+ @NotifyChange("presentedIntervals")
+ public void clearPresentedIntervals() {
+ intervals.clear();
+ presentedIntervals = null;
+ }
+
+ private void clearDatabaseParameters() {
+ schema = null;
+ table = null;
+ BindUtils.postNotifyChange(null, null, this, "schema");
+ BindUtils.postNotifyChange(null, null, this, "table");
+ }
+
+ private void clearSpecificProperties() {
+ chosenTimeBaseField = null;
+ uniqueFieldName = null;
+ BindUtils.postNotifyChange(null, null, this, "chosenTimeBaseField");
+ BindUtils.postNotifyChange(null, null, this, "uniqueFieldName");
+ }
+
+ private void clearEncodingParameters() {
+ currentField = null;
+ currentEncodingType = null;
+ currentApplicableEncodingTypes.clear();
+ encodingParameters.clear();
+ allEncodingParametersSet = false;
+ disabledNextEncodingTypeButton = true;
+ disabledFinishEncodingTypeButton = true;
+ allEncodingParametersSet = false;
+ BindUtils.postNotifyChange(null, null, this, "currentField");
+ BindUtils.postNotifyChange(null, null, this, "currentEncodingType");
+ BindUtils.postNotifyChange(null, null, this, "currentApplicableEncodingTypes");
+ BindUtils.postNotifyChange(null, null, this, "disabledNextEncodingTypeButton");
+ BindUtils.postNotifyChange(null, null, this, "disabledFinishEncodingTypeButton");
+ }
+
+ private void clearFeatureEngineeringParameters() {
+ enabledFeatureEngineering = false;
+ retroSpectiveDay = null;
+ chosenFeatureField = null;
+ presentedIntervals = null;
+ intervals.clear();
+ featureEngineeringErrorMessage = null;
+ BindUtils.postNotifyChange(null, null, this, "enabledFeatureEngineering");
+ BindUtils.postNotifyChange(null, null, this, "retroSpectiveDay");
+ BindUtils.postNotifyChange(null, null, this, "chosenFeatureField");
+ BindUtils.postNotifyChange(null, null, this, "presentedIntervals");
+ BindUtils.postNotifyChange(null, null, this, "featureEngineeringErrorMessage");
+ }
+
+ private void filterEncodingParameters() {
+ List<FieldProperty> fieldProperties = tableProperties.getFields();
+ for (FieldProperty property : fieldProperties) {
+ String fieldName = property.getName();
+ if (property.getType().equals(INT) || property.getType().equals(FLOAT) && encodingParameters.containsKey(
+ fieldName)) {
+ if (encodingParameters.get(fieldName).equals(INT) || encodingParameters.get(fieldName).equals(FLOAT)) {
+ encodingParameters.remove(fieldName);
+ }
+ }
+ }
+ }
+
+ public Validator getTrainTaskNameValidator() {
+ return new AbstractValidator() {
+ @Override
+ public void validate(ValidationContext validationContext) {
+ String plannedTrainTaskName = (String) validationContext.getProperty().getValue();
+ Map<Integer, TrainTaskParameters> trainTaskParametersById =
+ trainClient.getTrainTask(Collections.emptyMap());
+ List<String> trainTaskNames =
+ trainTaskParametersById.entrySet().stream().map(e -> e.getValue().getTrainTaskName())
+ .collect(Collectors.toList());
+ if (trainTaskNames.contains(plannedTrainTaskName)) {
+ addInvalidMessage(validationContext, "This name already exists");
+ }
+ }
+ };
+ }
+}
--- /dev/null
+package hu.user.trainmanager.viewmodel;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.zkoss.bind.annotation.BindingParam;
+import org.zkoss.bind.annotation.Command;
+import org.zkoss.bind.annotation.GlobalCommand;
+import org.zkoss.bind.annotation.Init;
+import org.zkoss.bind.annotation.NotifyChange;
+import org.zkoss.zk.ui.select.annotation.VariableResolver;
+import org.zkoss.zk.ui.select.annotation.Wire;
+import org.zkoss.zk.ui.select.annotation.WireVariable;
+import org.zkoss.zul.Filedownload;
+import org.zkoss.zul.Messagebox;
+
+import hu.user.trainmanager.dto.DirectoryProperty;
+import hu.user.trainmanager.dto.FileProperty;
+import hu.user.trainmanager.dto.csv.CsvContentDto;
+import hu.user.trainmanager.dto.csv.CsvDataHolder;
+import hu.user.trainmanager.service.CsvHandler;
+import hu.user.trainmanager.service.MetricsCsvDataBuilder;
+import hu.user.trainmanager.service.TrainClient;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@VariableResolver(org.zkoss.zkplus.spring.DelegatingVariableResolver.class)
+public class MetricsViewModel {
+ private String homeDirectory;
+ @WireVariable private TrainClient trainClient;
+ @WireVariable private MetricsCsvDataBuilder metricsCsvDataBuilder;
+ @WireVariable private CsvHandler csvHandler;
+
+// private List<EstimatorMetricsDto> estimatorMetrics = new ArrayList<>();
+
+ private String modalWindow;
+ private String fullPath;
+ private List<DirectoryProperty> directoryProperties = new ArrayList<>();
+ private List<FileProperty> fileProperties = new ArrayList<>();
+ private String trainTaskCsvFileName;
+ private String metricsCsvFileName;
+ private boolean enabledTrainTaskCsvDateTimeSuffix;
+ private boolean enabledMetricsCsvDateTimeSuffix;
+ private File selectedDirectory;
+ private CsvContentDto metricsCsvContentDto;
+
+ @Init
+ public void init() {
+ homeDirectory = trainClient.getHomeDirectory();
+ File root = new File(homeDirectory);
+ selectedDirectory = root;
+ directoryProperties = getDirectoryProperties(root);
+ fullPath = root.getAbsolutePath();
+ metricsCsvContentDto = trainClient.getMetricsCsvContent();
+ }
+
+ @Command
+ @GlobalCommand
+ @NotifyChange("metricsCsvContentDto")
+ public void refreshMetrics() {
+ metricsCsvContentDto = trainClient.getMetricsCsvContent();
+ }
+
+
+// @Command
+// @NotifyChange("modalWindow")
+// public void showSaveMetricsWindow() {
+// modalWindow = "~./zul/metricsSave.zul";
+// System.out.println(modalWindow);
+// }
+
+ @Command
+ public void saveMetricsCsv() {
+ System.out.println("on");
+ CsvDataHolder csvDataHolder = metricsCsvDataBuilder.getCsvContent();
+ String filePath = "/home/tomi/log/csv.log";
+ csvHandler.writeDataLineByLine(filePath, csvDataHolder);
+ }
+
+// @Command
+// public void save() {
+// try {
+// File file = new File("/home/tomi/tmp/AWS_Konyv.pdf");
+// Filedownload.save(file, "application/pdf");
+// } catch (FileNotFoundException e) {
+// System.out.println("Not found");
+// }
+// }
+
+ @Command
+ @NotifyChange({"directoryProperties", "fileProperties", "fullPath"})
+ public void changeDirectory(@BindingParam("param") String selectedDirectoryName) {
+ selectedDirectory = new File(selectedDirectoryName);
+ directoryProperties.clear();
+ directoryProperties.add(new DirectoryProperty("..", selectedDirectory.getParentFile()));
+ directoryProperties.addAll(getDirectoryProperties(selectedDirectory));
+ fileProperties = getFileProperties(selectedDirectory);
+ fullPath = selectedDirectoryName;
+ }
+
+ @Command
+ public void downloadSelectedCsvFile(@BindingParam("param") String selectedFileName) {
+ try {
+ File selectedFile = new File(selectedFileName);
+ Filedownload.save(selectedFile, null);
+ } catch (FileNotFoundException e) {
+ System.out.println("Not found");
+ }
+ }
+
+ @Command
+ @NotifyChange("fileProperties")
+ public void deleteSelectedCsvFile(@BindingParam("param") String selectedFileName) {
+ File selectedFile = new File(selectedFileName);
+ try {
+ boolean result = Files.deleteIfExists(Paths.get(selectedFileName));
+ if (result) {
+ Messagebox.show("File deleted");
+ } else {
+ Messagebox.show("Unable to delete the file");
+ }
+ } catch (IOException e) {
+ System.out.println("exception");
+ }
+ Optional<FileProperty> optionalFileProperty =
+ fileProperties.stream().filter(e -> e.getFile().equals(selectedFile)).findFirst();
+ optionalFileProperty.ifPresent(e -> fileProperties.remove(e));
+ }
+
+ @Command
+ @NotifyChange({"fileProperties", "trainTaskCsvFileName"})
+ public void createTrainTaskCsvFile() {
+ csvHandler.createTrainTaskCsvFile(fullPath, trainTaskCsvFileName, enabledTrainTaskCsvDateTimeSuffix);
+ fileProperties = getFileProperties(selectedDirectory);
+ trainTaskCsvFileName = "";
+ }
+
+ @Command
+ @NotifyChange({"fileProperties", "metricsCsvFileName"})
+ public void createMetricsCsvFile() {
+ csvHandler.createMetricsCsvFile(fullPath, metricsCsvFileName, enabledMetricsCsvDateTimeSuffix);
+ fileProperties = getFileProperties(selectedDirectory);
+ metricsCsvFileName = "";
+
+ }
+
+ private List<DirectoryProperty> getDirectoryProperties(File directory) {
+ return Arrays.stream(directory.listFiles()).filter(e -> e.isDirectory() && !e.isHidden())
+ .map(e -> new DirectoryProperty(e.getName(), new File(e.toURI()))).collect(Collectors.toList());
+ }
+
+ private List<FileProperty> getFileProperties(File directory) {
+ return Arrays.stream(directory.listFiles()).filter(e -> e.isFile()).map(e -> new FileProperty(e, e.getName()))
+ .collect(Collectors.toList());
+ }
+}
--- /dev/null
+package hu.user.trainmanager.viewmodel;
+
+import hu.user.trainmanager.event.TrainTaskEvent;
+
+public interface ProgressPresenter {
+
+ void changeUi(TrainTaskEvent event);
+}
--- /dev/null
+restart.include.zklibs=/z[\\w]+-[\\w\\d-\.]+\.jar
--- /dev/null
+server.port=8084
+zk.homepage=start
+zk.zul-view-resolver-prefix=/zul
+spring.devtools.restart.exclude=web/zul/**
+home.directory=/home/tomi
+evaluation.csv.directory=/home/tomi/csv
--- /dev/null
+spring:
+ profiles:
+ default: "dev"
+ application:
+ name: TrainManager
+ datasource:
+ url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/common_fraud?serverTimezone=UTC
+ username: root
+ password: pwd
+logging:
+ level:
+ root: info
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /home/tomi/log/${spring.application.name}.log
+server:
+ port: 8086
+trainUrl: http://localhost:8085
+showProgressRepeatTime: 500
+trainLivenessRepeatTime: 100
+
+---
+spring:
+ config:
+ activate:
+ on-profile: dockerized
+# jpa:
+# show-sql: true
+# generate-ddl: true
+# hibernate:
+# formar_sql: true
+logging:
+ pattern:
+ console: = %d{yyyy-MM-dd HH:mm:ss} - %msg%n
+ file: =%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ file:
+ name: /opt/app/log/${spring.application.name}.log
+server:
+ port: 8086
+trainUrl: http://train:8085
+showProgressRepeatTime: 500
+trainLivenessRepeatTime: 100
+
+---
\ No newline at end of file
--- /dev/null
+<language-addon>
+ <addon-name>zkspringboot-demo-addon</addon-name>
+ <version>1.0</version>
+ <language-name>xul/html</language-name>
+
+ <depends>zkmax</depends>
+
+ <stylesheet href="/css/static-globalstyles.css" type="text/css"/>
+ <stylesheet href="~./css/web-globalstyles.css" type="text/css"/>
+</language-addon>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<zk>
+ <system-config>
+ <disable-zscript>false</disable-zscript>
+ </system-config>
+ <!-- show debug information -->
+ <client-config>
+ <debug-js>true</debug-js>
+ </client-config>
+ <library-property>
+ <name>org.zkoss.zk.ui.versionInfo.enabled</name>
+ <value>true</value>
+ </library-property>
+ <!-- disable various caches -->
+ <library-property>
+ <name>org.zkoss.zk.ZUML.cache</name>
+ <value>false</value>
+ </library-property>
+ <library-property>
+ <name>org.zkoss.zk.WPD.cache</name>
+ <value>false</value>
+ </library-property>
+ <library-property>
+ <name>org.zkoss.zk.WCS.cache</name>
+ <value>false</value>
+ </library-property>
+ <library-property>
+ <name>org.zkoss.web.classWebResource.cache</name>
+ <value>false</value>
+ </library-property>
+ <library-property>
+ <name>org.zkoss.util.label.cache</name>
+ <value>false</value>
+ </library-property>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<zk>
+ <config-name>zkspringboot-demo</config-name>
+ <library-property>
+ <name>org.zkoss.zul.nativebar</name>
+ <value>true</value>
+ </library-property>
+ <library-property>
+ <name>org.zkoss.zul.image.preload</name>
+ <value>true</value>
+ </library-property>
+<!-- <library-property>-->
+<!-- <name>org.zkoss.zk.config.path</name>-->
+<!-- <value>zk-dev.xml</value>-->
+<!-- </library-property>-->
+
+ <!--enable websocket support in ZK 8.5.1 (ZK-3799) -->
+ <!-- <listener>-->
+ <!-- <listener-class>org.zkoss.zkmax.au.websocket.WebSocketWebAppInit</listener-class>-->
+ <!-- </listener>-->
+
+ <!-- <richlet>-->
+ <!-- <richlet-name>ExampleRichlet</richlet-name><!– your preferred name –>-->
+ <!-- <richlet-class>org.zkoss.zkspringboot.demo.richlet.ExampleRichlet</richlet-class><!– your class name, of course –>-->
+ <!-- </richlet>-->
+ <!-- <richlet-mapping>-->
+ <!-- <richlet-name>ExampleRichlet</richlet-name>-->
+ <!-- <url-pattern>/richlet/example</url-pattern>-->
+ <!-- </richlet-mapping>-->
+</zk>
--- /dev/null
+.bg {
+ background-image: url("plane.jpg");
+ background-size: cover;
+}
\ No newline at end of file
--- /dev/null
+/* Created with Themestr.app */
+/*! `Custom` Bootstrap 4 theme *//*!
+ * Bootstrap v4.3.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 The Bootstrap Authors
+ * Copyright 2011-2019 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */:root{--blue: #007bff;--indigo: #6610f2;--purple: #6f42c1;--pink: #e83e8c;--red: #dc3545;--orange: #fd7e14;--yellow: #ffc107;--green: #28a745;--teal: #20c997;--cyan: #17a2b8;--white: #fff;--gray: #6c757d;--gray-dark: #343a40;--primary: #0093f9;--secondary: #B2DEFD;--success: #15CAB4;--info: #9665ed;--warning: #ffa516;--danger: #ff4051;--light: #f8f9fa;--dark: #343a40;--breakpoint-xs: 0;--breakpoint-sm: 576px;--breakpoint-md: 768px;--breakpoint-lg: 992px;--breakpoint-xl: 1200px;--font-family-sans-serif: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";--font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace}*,*::before,*::after{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:rgba(0,0,0,0)}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0 !important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[title],abbr[data-original-title]{text-decoration:underline;text-decoration:underline dotted;cursor:help;border-bottom:0;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul,dl{margin-top:0;margin-bottom:1rem}ol ol,ul ul,ol ul,ul ol{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#0093f9;text-decoration:none;background-color:transparent}a:hover{color:#0066ad;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):hover,a:not([href]):not([tabindex]):focus{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}pre,code,kbd,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}input,button,select,optgroup,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}select{word-wrap:normal}button,[type="button"],[type="reset"],[type="submit"]{-webkit-appearance:button}button:not(:disabled),[type="button"]:not(:disabled),[type="reset"]:not(:disabled),[type="submit"]:not(:disabled){cursor:pointer}button::-moz-focus-inner,[type="button"]::-moz-focus-inner,[type="reset"]::-moz-focus-inner,[type="submit"]::-moz-focus-inner{padding:0;border-style:none}input[type="radio"],input[type="checkbox"]{box-sizing:border-box;padding:0}input[type="date"],input[type="time"],input[type="datetime-local"],input[type="month"]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type="number"]::-webkit-inner-spin-button,[type="number"]::-webkit-outer-spin-button{height:auto}[type="search"]{outline-offset:-2px;-webkit-appearance:none}[type="search"]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none !important}h1,h2,h3,h4,h5,h6,.h1,.h2,.h3,.h4,.h5,.h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}h1,.h1{font-size:2.5rem}h2,.h2{font-size:2rem}h3,.h3{font-size:1.75rem}h4,.h4{font-size:1.5rem}h5,.h5{font-size:1.25rem}h6,.h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,0.1)}small,.small{font-size:80%;font-weight:400}mark,.mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-break:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.table{width:100%;margin-bottom:1rem;color:#212529}.table th,.table td{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm th,.table-sm td{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered th,.table-bordered td{border:1px solid #dee2e6}.table-bordered thead th,.table-bordered thead td{border-bottom-width:2px}.table-borderless th,.table-borderless td,.table-borderless thead th,.table-borderless tbody+tbody{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,0.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,0.075)}.table-primary,.table-primary>th,.table-primary>td{background-color:#b8e1fd}.table-primary th,.table-primary td,.table-primary thead th,.table-primary tbody+tbody{border-color:#7ac7fc}.table-hover .table-primary:hover{background-color:#9fd7fc}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fd7fc}.table-secondary,.table-secondary>th,.table-secondary>td{background-color:#e9f6fe}.table-secondary th,.table-secondary td,.table-secondary thead th,.table-secondary tbody+tbody{border-color:#d7eefe}.table-hover .table-secondary:hover{background-color:#d1ecfd}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#d1ecfd}.table-success,.table-success>th,.table-success>td{background-color:#bdf0ea}.table-success th,.table-success td,.table-success thead th,.table-success tbody+tbody{border-color:#85e3d8}.table-hover .table-success:hover{background-color:#a8ebe3}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#a8ebe3}.table-info,.table-info>th,.table-info>td{background-color:#e2d4fa}.table-info th,.table-info td,.table-info thead th,.table-info tbody+tbody{border-color:#c8aff6}.table-hover .table-info:hover{background-color:#d3bdf7}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#d3bdf7}.table-warning,.table-warning>th,.table-warning>td{background-color:#ffe6be}.table-warning th,.table-warning td,.table-warning thead th,.table-warning tbody+tbody{border-color:#ffd086}.table-hover .table-warning:hover{background-color:#ffdca5}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffdca5}.table-danger,.table-danger>th,.table-danger>td{background-color:#ffcace}.table-danger th,.table-danger td,.table-danger thead th,.table-danger tbody+tbody{border-color:#ff9ca5}.table-hover .table-danger:hover{background-color:#ffb1b6}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#ffb1b6}.table-light,.table-light>th,.table-light>td{background-color:#fdfdfe}.table-light th,.table-light td,.table-light thead th,.table-light tbody+tbody{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>th,.table-dark>td{background-color:#c6c8ca}.table-dark th,.table-dark td,.table-dark thead th,.table-dark tbody+tbody{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>th,.table-active>td{background-color:rgba(0,0,0,0.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,0.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,0.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark th,.table-dark td,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,0.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,0.075)}@media (max-width: 575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width: 767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width: 991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width: 1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#495057;background-color:#fff;border-color:#7ac8ff;outline:0;box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding-top:.375rem;padding-bottom:.375rem;margin-bottom:0;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-sm,.form-control-plaintext.form-control-lg{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[size],select.form-control[multiple]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:flex;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*="col-"]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled ~ .form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:inline-flex;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#15CAB4}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(21,202,180,0.9);border-radius:.25rem}.was-validated .form-control:valid,.form-control.is-valid{border-color:#15CAB4;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%2315CAB4' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:center right calc(.375em + .1875rem);background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-control:valid:focus,.form-control.is-valid:focus{border-color:#15CAB4;box-shadow:0 0 0 .2rem rgba(21,202,180,0.25)}.was-validated .form-control:valid ~ .valid-feedback,.was-validated .form-control:valid ~ .valid-tooltip,.form-control.is-valid ~ .valid-feedback,.form-control.is-valid ~ .valid-tooltip{display:block}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.was-validated .custom-select:valid,.custom-select.is-valid{border-color:#15CAB4;padding-right:calc((1em + .75rem) * 3 / 4 + 1.75rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%2315CAB4' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .custom-select:valid:focus,.custom-select.is-valid:focus{border-color:#15CAB4;box-shadow:0 0 0 .2rem rgba(21,202,180,0.25)}.was-validated .custom-select:valid ~ .valid-feedback,.was-validated .custom-select:valid ~ .valid-tooltip,.custom-select.is-valid ~ .valid-feedback,.custom-select.is-valid ~ .valid-tooltip{display:block}.was-validated .form-control-file:valid ~ .valid-feedback,.was-validated .form-control-file:valid ~ .valid-tooltip,.form-control-file.is-valid ~ .valid-feedback,.form-control-file.is-valid ~ .valid-tooltip{display:block}.was-validated .form-check-input:valid ~ .form-check-label,.form-check-input.is-valid ~ .form-check-label{color:#15CAB4}.was-validated .form-check-input:valid ~ .valid-feedback,.was-validated .form-check-input:valid ~ .valid-tooltip,.form-check-input.is-valid ~ .valid-feedback,.form-check-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-control-input:valid ~ .custom-control-label,.custom-control-input.is-valid ~ .custom-control-label{color:#15CAB4}.was-validated .custom-control-input:valid ~ .custom-control-label::before,.custom-control-input.is-valid ~ .custom-control-label::before{border-color:#15CAB4}.was-validated .custom-control-input:valid ~ .valid-feedback,.was-validated .custom-control-input:valid ~ .valid-tooltip,.custom-control-input.is-valid ~ .valid-feedback,.custom-control-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-control-input:valid:checked ~ .custom-control-label::before,.custom-control-input.is-valid:checked ~ .custom-control-label::before{border-color:#29e9d1;background-color:#29e9d1}.was-validated .custom-control-input:valid:focus ~ .custom-control-label::before,.custom-control-input.is-valid:focus ~ .custom-control-label::before{box-shadow:0 0 0 .2rem rgba(21,202,180,0.25)}.was-validated .custom-control-input:valid:focus:not(:checked) ~ .custom-control-label::before,.custom-control-input.is-valid:focus:not(:checked) ~ .custom-control-label::before{border-color:#15CAB4}.was-validated .custom-file-input:valid ~ .custom-file-label,.custom-file-input.is-valid ~ .custom-file-label{border-color:#15CAB4}.was-validated .custom-file-input:valid ~ .valid-feedback,.was-validated .custom-file-input:valid ~ .valid-tooltip,.custom-file-input.is-valid ~ .valid-feedback,.custom-file-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-file-input:valid:focus ~ .custom-file-label,.custom-file-input.is-valid:focus ~ .custom-file-label{border-color:#15CAB4;box-shadow:0 0 0 .2rem rgba(21,202,180,0.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#ff4051}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(255,64,81,0.9);border-radius:.25rem}.was-validated .form-control:invalid,.form-control.is-invalid{border-color:#ff4051;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23ff4051' viewBox='-2 -2 7 7'%3e%3cpath stroke='%23ff4051' d='M0 0l3 3m0-3L0 3'/%3e%3ccircle r='.5'/%3e%3ccircle cx='3' r='.5'/%3e%3ccircle cy='3' r='.5'/%3e%3ccircle cx='3' cy='3' r='.5'/%3e%3c/svg%3E");background-repeat:no-repeat;background-position:center right calc(.375em + .1875rem);background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-control:invalid:focus,.form-control.is-invalid:focus{border-color:#ff4051;box-shadow:0 0 0 .2rem rgba(255,64,81,0.25)}.was-validated .form-control:invalid ~ .invalid-feedback,.was-validated .form-control:invalid ~ .invalid-tooltip,.form-control.is-invalid ~ .invalid-feedback,.form-control.is-invalid ~ .invalid-tooltip{display:block}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.was-validated .custom-select:invalid,.custom-select.is-invalid{border-color:#ff4051;padding-right:calc((1em + .75rem) * 3 / 4 + 1.75rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23ff4051' viewBox='-2 -2 7 7'%3e%3cpath stroke='%23ff4051' d='M0 0l3 3m0-3L0 3'/%3e%3ccircle r='.5'/%3e%3ccircle cx='3' r='.5'/%3e%3ccircle cy='3' r='.5'/%3e%3ccircle cx='3' cy='3' r='.5'/%3e%3c/svg%3E") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .custom-select:invalid:focus,.custom-select.is-invalid:focus{border-color:#ff4051;box-shadow:0 0 0 .2rem rgba(255,64,81,0.25)}.was-validated .custom-select:invalid ~ .invalid-feedback,.was-validated .custom-select:invalid ~ .invalid-tooltip,.custom-select.is-invalid ~ .invalid-feedback,.custom-select.is-invalid ~ .invalid-tooltip{display:block}.was-validated .form-control-file:invalid ~ .invalid-feedback,.was-validated .form-control-file:invalid ~ .invalid-tooltip,.form-control-file.is-invalid ~ .invalid-feedback,.form-control-file.is-invalid ~ .invalid-tooltip{display:block}.was-validated .form-check-input:invalid ~ .form-check-label,.form-check-input.is-invalid ~ .form-check-label{color:#ff4051}.was-validated .form-check-input:invalid ~ .invalid-feedback,.was-validated .form-check-input:invalid ~ .invalid-tooltip,.form-check-input.is-invalid ~ .invalid-feedback,.form-check-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-control-input:invalid ~ .custom-control-label,.custom-control-input.is-invalid ~ .custom-control-label{color:#ff4051}.was-validated .custom-control-input:invalid ~ .custom-control-label::before,.custom-control-input.is-invalid ~ .custom-control-label::before{border-color:#ff4051}.was-validated .custom-control-input:invalid ~ .invalid-feedback,.was-validated .custom-control-input:invalid ~ .invalid-tooltip,.custom-control-input.is-invalid ~ .invalid-feedback,.custom-control-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-control-input:invalid:checked ~ .custom-control-label::before,.custom-control-input.is-invalid:checked ~ .custom-control-label::before{border-color:#ff737f;background-color:#ff737f}.was-validated .custom-control-input:invalid:focus ~ .custom-control-label::before,.custom-control-input.is-invalid:focus ~ .custom-control-label::before{box-shadow:0 0 0 .2rem rgba(255,64,81,0.25)}.was-validated .custom-control-input:invalid:focus:not(:checked) ~ .custom-control-label::before,.custom-control-input.is-invalid:focus:not(:checked) ~ .custom-control-label::before{border-color:#ff4051}.was-validated .custom-file-input:invalid ~ .custom-file-label,.custom-file-input.is-invalid ~ .custom-file-label{border-color:#ff4051}.was-validated .custom-file-input:invalid ~ .invalid-feedback,.was-validated .custom-file-input:invalid ~ .invalid-tooltip,.custom-file-input.is-invalid ~ .invalid-feedback,.custom-file-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-file-input:invalid:focus ~ .custom-file-label,.custom-file-input.is-invalid:focus ~ .custom-file-label{border-color:#ff4051;box-shadow:0 0 0 .2rem rgba(255,64,81,0.25)}.form-inline{display:flex;flex-flow:row wrap;align-items:center}.form-inline .form-check{width:100%}@media (min-width: 576px){.form-inline label{display:flex;align-items:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:flex;flex:0 0 auto;flex-flow:row wrap;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .input-group,.form-inline .custom-select{width:auto}.form-inline .form-check{display:flex;align-items:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{align-items:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color 0.15s ease-in-out,background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out}@media (prefers-reduced-motion: reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn:focus,.btn.focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.btn.disabled,.btn:disabled{opacity:.65}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#0093f9;border-color:#0093f9}.btn-primary:hover{color:#fff;background-color:#007cd3;border-color:#0075c6}.btn-primary:focus,.btn-primary.focus{box-shadow:0 0 0 .2rem rgba(38,163,250,0.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#0093f9;border-color:#0093f9}.btn-primary:not(:disabled):not(.disabled):active,.btn-primary:not(:disabled):not(.disabled).active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0075c6;border-color:#006db9}.btn-primary:not(:disabled):not(.disabled):active:focus,.btn-primary:not(:disabled):not(.disabled).active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,163,250,0.5)}.btn-secondary{color:#212529;background-color:#B2DEFD;border-color:#B2DEFD}.btn-secondary:hover{color:#212529;background-color:#8dcefc;border-color:#80c9fc}.btn-secondary:focus,.btn-secondary.focus{box-shadow:0 0 0 .2rem rgba(156,194,221,0.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#212529;background-color:#B2DEFD;border-color:#B2DEFD}.btn-secondary:not(:disabled):not(.disabled):active,.btn-secondary:not(:disabled):not(.disabled).active,.show>.btn-secondary.dropdown-toggle{color:#212529;background-color:#80c9fc;border-color:#74c3fb}.btn-secondary:not(:disabled):not(.disabled):active:focus,.btn-secondary:not(:disabled):not(.disabled).active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(156,194,221,0.5)}.btn-success{color:#fff;background-color:#15CAB4;border-color:#15CAB4}.btn-success:hover{color:#fff;background-color:#11a795;border-color:#109c8b}.btn-success:focus,.btn-success.focus{box-shadow:0 0 0 .2rem rgba(56,210,191,0.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#15CAB4;border-color:#15CAB4}.btn-success:not(:disabled):not(.disabled):active,.btn-success:not(:disabled):not(.disabled).active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#109c8b;border-color:#0f9081}.btn-success:not(:disabled):not(.disabled):active:focus,.btn-success:not(:disabled):not(.disabled).active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(56,210,191,0.5)}.btn-info{color:#fff;background-color:#9665ed;border-color:#9665ed}.btn-info:hover{color:#fff;background-color:#7f43e9;border-color:#7737e8}.btn-info:focus,.btn-info.focus{box-shadow:0 0 0 .2rem rgba(166,124,240,0.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#9665ed;border-color:#9665ed}.btn-info:not(:disabled):not(.disabled):active,.btn-info:not(:disabled):not(.disabled).active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#7737e8;border-color:#6f2ce6}.btn-info:not(:disabled):not(.disabled):active:focus,.btn-info:not(:disabled):not(.disabled).active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(166,124,240,0.5)}.btn-warning{color:#212529;background-color:#ffa516;border-color:#ffa516}.btn-warning:hover{color:#212529;background-color:#ef9300;border-color:#e28b00}.btn-warning:focus,.btn-warning.focus{box-shadow:0 0 0 .2rem rgba(222,146,25,0.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffa516;border-color:#ffa516}.btn-warning:not(:disabled):not(.disabled):active,.btn-warning:not(:disabled):not(.disabled).active,.show>.btn-warning.dropdown-toggle{color:#fff;background-color:#e28b00;border-color:#d58300}.btn-warning:not(:disabled):not(.disabled):active:focus,.btn-warning:not(:disabled):not(.disabled).active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,146,25,0.5)}.btn-danger{color:#fff;background-color:#ff4051;border-color:#ff4051}.btn-danger:hover{color:#fff;background-color:#ff1a2e;border-color:#ff0d23}.btn-danger:focus,.btn-danger.focus{box-shadow:0 0 0 .2rem rgba(255,93,107,0.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#ff4051;border-color:#ff4051}.btn-danger:not(:disabled):not(.disabled):active,.btn-danger:not(:disabled):not(.disabled).active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#ff0d23;border-color:#ff0017}.btn-danger:not(:disabled):not(.disabled):active:focus,.btn-danger:not(:disabled):not(.disabled).active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,93,107,0.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light:focus,.btn-light.focus{box-shadow:0 0 0 .2rem rgba(216,217,219,0.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled):active,.btn-light:not(:disabled):not(.disabled).active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled):active:focus,.btn-light:not(:disabled):not(.disabled).active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,0.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark:focus,.btn-dark.focus{box-shadow:0 0 0 .2rem rgba(82,88,93,0.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled):active,.btn-dark:not(:disabled):not(.disabled).active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled):active:focus,.btn-dark:not(:disabled):not(.disabled).active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,0.5)}.btn-outline-primary{color:#0093f9;border-color:#0093f9}.btn-outline-primary:hover{color:#fff;background-color:#0093f9;border-color:#0093f9}.btn-outline-primary:focus,.btn-outline-primary.focus{box-shadow:0 0 0 .2rem rgba(0,147,249,0.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#0093f9;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled):active,.btn-outline-primary:not(:disabled):not(.disabled).active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#0093f9;border-color:#0093f9}.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,147,249,0.5)}.btn-outline-secondary{color:#B2DEFD;border-color:#B2DEFD}.btn-outline-secondary:hover{color:#212529;background-color:#B2DEFD;border-color:#B2DEFD}.btn-outline-secondary:focus,.btn-outline-secondary.focus{box-shadow:0 0 0 .2rem rgba(178,222,253,0.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#B2DEFD;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled):active,.btn-outline-secondary:not(:disabled):not(.disabled).active,.show>.btn-outline-secondary.dropdown-toggle{color:#212529;background-color:#B2DEFD;border-color:#B2DEFD}.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(178,222,253,0.5)}.btn-outline-success{color:#15CAB4;border-color:#15CAB4}.btn-outline-success:hover{color:#fff;background-color:#15CAB4;border-color:#15CAB4}.btn-outline-success:focus,.btn-outline-success.focus{box-shadow:0 0 0 .2rem rgba(21,202,180,0.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#15CAB4;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled):active,.btn-outline-success:not(:disabled):not(.disabled).active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#15CAB4;border-color:#15CAB4}.btn-outline-success:not(:disabled):not(.disabled):active:focus,.btn-outline-success:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(21,202,180,0.5)}.btn-outline-info{color:#9665ed;border-color:#9665ed}.btn-outline-info:hover{color:#fff;background-color:#9665ed;border-color:#9665ed}.btn-outline-info:focus,.btn-outline-info.focus{box-shadow:0 0 0 .2rem rgba(150,101,237,0.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#9665ed;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled):active,.btn-outline-info:not(:disabled):not(.disabled).active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#9665ed;border-color:#9665ed}.btn-outline-info:not(:disabled):not(.disabled):active:focus,.btn-outline-info:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(150,101,237,0.5)}.btn-outline-warning{color:#ffa516;border-color:#ffa516}.btn-outline-warning:hover{color:#212529;background-color:#ffa516;border-color:#ffa516}.btn-outline-warning:focus,.btn-outline-warning.focus{box-shadow:0 0 0 .2rem rgba(255,165,22,0.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffa516;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled):active,.btn-outline-warning:not(:disabled):not(.disabled).active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffa516;border-color:#ffa516}.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,165,22,0.5)}.btn-outline-danger{color:#ff4051;border-color:#ff4051}.btn-outline-danger:hover{color:#fff;background-color:#ff4051;border-color:#ff4051}.btn-outline-danger:focus,.btn-outline-danger.focus{box-shadow:0 0 0 .2rem rgba(255,64,81,0.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#ff4051;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled):active,.btn-outline-danger:not(:disabled):not(.disabled).active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#ff4051;border-color:#ff4051}.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,64,81,0.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:focus,.btn-outline-light.focus{box-shadow:0 0 0 .2rem rgba(248,249,250,0.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled):active,.btn-outline-light:not(:disabled):not(.disabled).active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled):active:focus,.btn-outline-light:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,0.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:focus,.btn-outline-dark.focus{box-shadow:0 0 0 .2rem rgba(52,58,64,0.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled):active,.btn-outline-dark:not(:disabled):not(.disabled).active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,0.5)}.btn-link{font-weight:400;color:#0093f9;text-decoration:none}.btn-link:hover{color:#0066ad;text-decoration:underline}.btn-link:focus,.btn-link.focus{text-decoration:underline;box-shadow:none}.btn-link:disabled,.btn-link.disabled{color:#6c757d;pointer-events:none}.btn-lg,.btn-group-lg>.btn{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-sm,.btn-group-sm>.btn{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.fade{transition:opacity 0.15s linear}@media (prefers-reduced-motion: reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height 0.35s ease}@media (prefers-reduced-motion: reduce){.collapsing{transition:none}}.dropup,.dropright,.dropdown,.dropleft{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width: 576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width: 768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width: 992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width: 1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^="top"],.dropdown-menu[x-placement^="right"],.dropdown-menu[x-placement^="bottom"],.dropdown-menu[x-placement^="left"]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:hover,.dropdown-item:focus{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#0093f9}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;flex:1 1 auto}.btn-group>.btn:hover,.btn-group-vertical>.btn:hover{z-index:1}.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn.active{z-index:1}.btn-toolbar{display:flex;flex-wrap:wrap;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn:not(:first-child),.btn-group>.btn-group:not(:first-child){margin-left:-1px}.btn-group>.btn:not(:last-child):not(.dropdown-toggle),.btn-group>.btn-group:not(:last-child)>.btn{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:not(:first-child),.btn-group>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-sm+.dropdown-toggle-split,.btn-group-sm>.btn+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-lg+.dropdown-toggle-split,.btn-group-lg>.btn+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;align-items:flex-start;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn:not(:first-child),.btn-group-vertical>.btn-group:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle),.btn-group-vertical>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:not(:first-child),.btn-group-vertical>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type="radio"],.btn-group-toggle>.btn input[type="checkbox"],.btn-group-toggle>.btn-group>.btn input[type="radio"],.btn-group-toggle>.btn-group>.btn input[type="checkbox"]{position:absolute;clip:rect(0, 0, 0, 0);pointer-events:none}.input-group{position:relative;display:flex;flex-wrap:wrap;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-control-plaintext,.input-group>.custom-select,.input-group>.custom-file{position:relative;flex:1 1 auto;width:1%;margin-bottom:0}.input-group>.form-control+.form-control,.input-group>.form-control+.custom-select,.input-group>.form-control+.custom-file,.input-group>.form-control-plaintext+.form-control,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.custom-file,.input-group>.custom-select+.form-control,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.custom-file,.input-group>.custom-file+.form-control,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.custom-file{margin-left:-1px}.input-group>.form-control:focus,.input-group>.custom-select:focus,.input-group>.custom-file .custom-file-input:focus ~ .custom-file-label{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.form-control:not(:last-child),.input-group>.custom-select:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.form-control:not(:first-child),.input-group>.custom-select:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:flex;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-prepend,.input-group-append{display:flex}.input-group-prepend .btn,.input-group-append .btn{position:relative;z-index:2}.input-group-prepend .btn:focus,.input-group-append .btn:focus{z-index:3}.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.input-group-text,.input-group-append .input-group-text+.btn{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:flex;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type="radio"],.input-group-text input[type="checkbox"]{margin-top:0}.input-group-lg>.form-control:not(textarea),.input-group-lg>.custom-select{height:calc(1.5em + 1rem + 2px)}.input-group-lg>.form-control,.input-group-lg>.custom-select,.input-group-lg>.input-group-prepend>.input-group-text,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-append>.btn{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.form-control:not(textarea),.input-group-sm>.custom-select{height:calc(1.5em + .5rem + 2px)}.input-group-sm>.form-control,.input-group-sm>.custom-select,.input-group-sm>.input-group-prepend>.input-group-text,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-append>.btn{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text,.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked ~ .custom-control-label::before{color:#fff;border-color:#0093f9;background-color:#0093f9}.custom-control-input:focus ~ .custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.custom-control-input:focus:not(:checked) ~ .custom-control-label::before{border-color:#7ac8ff}.custom-control-input:not(:disabled):active ~ .custom-control-label::before{color:#fff;background-color:#adddff;border-color:#adddff}.custom-control-input:disabled ~ .custom-control-label{color:#6c757d}.custom-control-input:disabled ~ .custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50% / 50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked ~ .custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::before{border-color:#0093f9;background-color:#0093f9}.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked ~ .custom-control-label::before{background-color:rgba(0,147,249,0.5)}.custom-checkbox .custom-control-input:disabled:indeterminate ~ .custom-control-label::before{background-color:rgba(0,147,249,0.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked ~ .custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked ~ .custom-control-label::before{background-color:rgba(0,147,249,0.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:transform 0.15s ease-in-out,background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out}@media (prefers-reduced-motion: reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked ~ .custom-control-label::after{background-color:#fff;transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked ~ .custom-control-label::before{background-color:rgba(0,147,249,0.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem;appearance:none}.custom-select:focus{border-color:#7ac8ff;outline:0;box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus ~ .custom-file-label{border-color:#7ac8ff;box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.custom-file-input:disabled ~ .custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en) ~ .custom-file-label::after{content:"Browse"}.custom-file-input ~ .custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:calc(1rem + .4rem);padding:0;background-color:transparent;appearance:none}.custom-range:focus{outline:none}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,147,249,0.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,147,249,0.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,147,249,0.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#0093f9;border:0;border-radius:1rem;transition:background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out;appearance:none}@media (prefers-reduced-motion: reduce){.custom-range::-webkit-slider-thumb{transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#adddff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#0093f9;border:0;border-radius:1rem;transition:background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out;appearance:none}@media (prefers-reduced-motion: reduce){.custom-range::-moz-range-thumb{transition:none}}.custom-range::-moz-range-thumb:active{background-color:#adddff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#0093f9;border:0;border-radius:1rem;transition:background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out;appearance:none}@media (prefers-reduced-motion: reduce){.custom-range::-ms-thumb{transition:none}}.custom-range::-ms-thumb:active{background-color:#adddff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out}@media (prefers-reduced-motion: reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:flex;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:hover,.nav-link:focus{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:hover,.nav-tabs .nav-link:focus{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-link.active,.nav-tabs .nav-item.show .nav-link{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#0093f9}.nav-fill .nav-item{flex:1 1 auto;text-align:center}.nav-justified .nav-item{flex-basis:0;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;padding:.5rem 1rem}.navbar>.container,.navbar>.container-fluid{display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none}.navbar-nav{display:flex;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{flex-basis:100%;flex-grow:1;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:hover,.navbar-toggler:focus{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width: 575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 576px){.navbar-expand-sm{flex-flow:row nowrap;justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:flex !important;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width: 767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 768px){.navbar-expand-md{flex-flow:row nowrap;justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:flex !important;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width: 991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 992px){.navbar-expand-lg{flex-flow:row nowrap;justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:flex !important;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width: 1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 1200px){.navbar-expand-xl{flex-flow:row nowrap;justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:flex !important;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{flex-flow:row nowrap;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid{flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:flex !important;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,0.9)}.navbar-light .navbar-brand:hover,.navbar-light .navbar-brand:focus{color:rgba(0,0,0,0.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,0.5)}.navbar-light .navbar-nav .nav-link:hover,.navbar-light .navbar-nav .nav-link:focus{color:rgba(0,0,0,0.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,0.3)}.navbar-light .navbar-nav .show>.nav-link,.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .nav-link.active{color:rgba(0,0,0,0.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,0.5);border-color:rgba(0,0,0,0.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3e%3cpath stroke='rgba(0,0,0,0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,0.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,0.9)}.navbar-light .navbar-text a:hover,.navbar-light .navbar-text a:focus{color:rgba(0,0,0,0.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:hover,.navbar-dark .navbar-brand:focus{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,0.5)}.navbar-dark .navbar-nav .nav-link:hover,.navbar-dark .navbar-nav .nav-link:focus{color:rgba(255,255,255,0.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,0.25)}.navbar-dark .navbar-nav .show>.nav-link,.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .nav-link.active{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,0.5);border-color:rgba(255,255,255,0.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3e%3cpath stroke='rgba(255,255,255,0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,0.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:hover,.navbar-dark .navbar-text a:focus{color:#fff}.card{position:relative;display:flex;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,0.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group:first-child .list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-body{flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,0.03);border-bottom:1px solid rgba(0,0,0,0.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,0.03);border-top:1px solid rgba(0,0,0,0.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img{width:100%;border-radius:calc(.25rem - 1px)}.card-img-top{width:100%;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img-bottom{width:100%;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck{display:flex;flex-direction:column}.card-deck .card{margin-bottom:15px}@media (min-width: 576px){.card-deck{flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{display:flex;flex:1 0 0%;flex-direction:column;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group{display:flex;flex-direction:column}.card-group>.card{margin-bottom:15px}@media (min-width: 576px){.card-group{flex-flow:row wrap}.card-group>.card{flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-img-top,.card-group>.card:not(:last-child) .card-header{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-img-bottom,.card-group>.card:not(:last-child) .card-footer{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-img-top,.card-group>.card:not(:first-child) .card-header{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-img-bottom,.card-group>.card:not(:first-child) .card-footer{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width: 576px){.card-columns{column-count:3;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:first-of-type) .card-header:first-child{border-radius:0}.accordion>.card:not(:first-of-type):not(:last-of-type){border-bottom:0;border-radius:0}.accordion>.card:first-of-type{border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:last-of-type{border-top-left-radius:0;border-top-right-radius:0}.accordion>.card .card-header{margin-bottom:-1px}.breadcrumb{display:flex;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#0093f9;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0066ad;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:2;outline:0;box-shadow:0 0 0 .2rem rgba(0,147,249,0.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:1;color:#fff;background-color:#0093f9;border-color:#0093f9}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color 0.15s ease-in-out,background-color 0.15s ease-in-out,border-color 0.15s ease-in-out,box-shadow 0.15s ease-in-out}@media (prefers-reduced-motion: reduce){.badge{transition:none}}a.badge:hover,a.badge:focus{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#0093f9}a.badge-primary:hover,a.badge-primary:focus{color:#fff;background-color:#0075c6}a.badge-primary:focus,a.badge-primary.focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,147,249,0.5)}.badge-secondary{color:#212529;background-color:#B2DEFD}a.badge-secondary:hover,a.badge-secondary:focus{color:#212529;background-color:#80c9fc}a.badge-secondary:focus,a.badge-secondary.focus{outline:0;box-shadow:0 0 0 .2rem rgba(178,222,253,0.5)}.badge-success{color:#fff;background-color:#15CAB4}a.badge-success:hover,a.badge-success:focus{color:#fff;background-color:#109c8b}a.badge-success:focus,a.badge-success.focus{outline:0;box-shadow:0 0 0 .2rem rgba(21,202,180,0.5)}.badge-info{color:#fff;background-color:#9665ed}a.badge-info:hover,a.badge-info:focus{color:#fff;background-color:#7737e8}a.badge-info:focus,a.badge-info.focus{outline:0;box-shadow:0 0 0 .2rem rgba(150,101,237,0.5)}.badge-warning{color:#212529;background-color:#ffa516}a.badge-warning:hover,a.badge-warning:focus{color:#212529;background-color:#e28b00}a.badge-warning:focus,a.badge-warning.focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,165,22,0.5)}.badge-danger{color:#fff;background-color:#ff4051}a.badge-danger:hover,a.badge-danger:focus{color:#fff;background-color:#ff0d23}a.badge-danger:focus,a.badge-danger.focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,64,81,0.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:hover,a.badge-light:focus{color:#212529;background-color:#dae0e5}a.badge-light:focus,a.badge-light.focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,0.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:hover,a.badge-dark:focus{color:#fff;background-color:#1d2124}a.badge-dark:focus,a.badge-dark.focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,0.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width: 576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004c81;background-color:#cce9fe;border-color:#b8e1fd}.alert-primary hr{border-top-color:#9fd7fc}.alert-primary .alert-link{color:#002e4e}.alert-secondary{color:#5d7384;background-color:#f0f8ff;border-color:#e9f6fe}.alert-secondary hr{border-top-color:#d1ecfd}.alert-secondary .alert-link{color:#485966}.alert-success{color:#0b695e;background-color:#d0f4f0;border-color:#bdf0ea}.alert-success hr{border-top-color:#a8ebe3}.alert-success .alert-link{color:#063b35}.alert-info{color:#4e357b;background-color:#eae0fb;border-color:#e2d4fa}.alert-info hr{border-top-color:#d3bdf7}.alert-info .alert-link{color:#372657}.alert-warning{color:#85560b;background-color:#ffedd0;border-color:#ffe6be}.alert-warning hr{border-top-color:#ffdca5}.alert-warning .alert-link{color:#563807}.alert-danger{color:#85212a;background-color:#ffd9dc;border-color:#ffcace}.alert-danger hr{border-top-color:#ffb1b6}.alert-danger .alert-link{color:#5c171d}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:flex;height:1rem;overflow:hidden;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:flex;flex-direction:column;justify-content:center;color:#fff;text-align:center;white-space:nowrap;background-color:#0093f9;transition:width 0.6s ease}@media (prefers-reduced-motion: reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-size:1rem 1rem}.progress-bar-animated{animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion: reduce){.progress-bar-animated{animation:none}}.media{display:flex;align-items:flex-start}.media-body{flex:1}.list-group{display:flex;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:hover,.list-group-item-action:focus{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,0.125)}.list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#0093f9;border-color:#0093f9}.list-group-horizontal{flex-direction:row}.list-group-horizontal .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}@media (min-width: 576px){.list-group-horizontal-sm{flex-direction:row}.list-group-horizontal-sm .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-sm .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width: 768px){.list-group-horizontal-md{flex-direction:row}.list-group-horizontal-md .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-md .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width: 992px){.list-group-horizontal-lg{flex-direction:row}.list-group-horizontal-lg .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-lg .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width: 1200px){.list-group-horizontal-xl{flex-direction:row}.list-group-horizontal-xl .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-xl .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush .list-group-item:last-child{margin-bottom:-1px}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{margin-bottom:0;border-bottom:0}.list-group-item-primary{color:#004c81;background-color:#b8e1fd}.list-group-item-primary.list-group-item-action:hover,.list-group-item-primary.list-group-item-action:focus{color:#004c81;background-color:#9fd7fc}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004c81;border-color:#004c81}.list-group-item-secondary{color:#5d7384;background-color:#e9f6fe}.list-group-item-secondary.list-group-item-action:hover,.list-group-item-secondary.list-group-item-action:focus{color:#5d7384;background-color:#d1ecfd}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#5d7384;border-color:#5d7384}.list-group-item-success{color:#0b695e;background-color:#bdf0ea}.list-group-item-success.list-group-item-action:hover,.list-group-item-success.list-group-item-action:focus{color:#0b695e;background-color:#a8ebe3}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#0b695e;border-color:#0b695e}.list-group-item-info{color:#4e357b;background-color:#e2d4fa}.list-group-item-info.list-group-item-action:hover,.list-group-item-info.list-group-item-action:focus{color:#4e357b;background-color:#d3bdf7}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#4e357b;border-color:#4e357b}.list-group-item-warning{color:#85560b;background-color:#ffe6be}.list-group-item-warning.list-group-item-action:hover,.list-group-item-warning.list-group-item-action:focus{color:#85560b;background-color:#ffdca5}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#85560b;border-color:#85560b}.list-group-item-danger{color:#85212a;background-color:#ffcace}.list-group-item-danger.list-group-item-action:hover,.list-group-item-danger.list-group-item-action:focus{color:#85212a;background-color:#ffb1b6}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#85212a;border-color:#85212a}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:hover,.list-group-item-light.list-group-item-action:focus{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:hover,.list-group-item-dark.list-group-item-action:focus{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):hover,.close:not(:disabled):not(.disabled):focus{opacity:.75}button.close{padding:0;background-color:transparent;border:0;appearance:none}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,0.85);background-clip:padding-box;border:1px solid rgba(0,0,0,0.1);box-shadow:0 0.25rem 0.75rem rgba(0,0,0,0.1);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:flex;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,0.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,0.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:transform 0.3s ease-out;transform:translate(0, -50px)}@media (prefers-reduced-motion: reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal-dialog-scrollable{display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-header,.modal-dialog-scrollable .modal-footer{flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);content:""}.modal-dialog-centered.modal-dialog-scrollable{flex-direction:column;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:flex;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:flex;align-items:flex-start;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:.3rem;border-top-right-radius:.3rem}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;flex:1 1 auto;padding:1rem}.modal-footer{display:flex;align-items:center;justify-content:flex-end;padding:1rem;border-top:1px solid #dee2e6;border-bottom-right-radius:.3rem;border-bottom-left-radius:.3rem}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width: 576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem)}.modal-sm{max-width:300px}}@media (min-width: 992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width: 1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-top,.bs-tooltip-auto[x-placement^="top"]{padding:.4rem 0}.bs-tooltip-top .arrow,.bs-tooltip-auto[x-placement^="top"] .arrow{bottom:0}.bs-tooltip-top .arrow::before,.bs-tooltip-auto[x-placement^="top"] .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-right,.bs-tooltip-auto[x-placement^="right"]{padding:0 .4rem}.bs-tooltip-right .arrow,.bs-tooltip-auto[x-placement^="right"] .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-right .arrow::before,.bs-tooltip-auto[x-placement^="right"] .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-bottom,.bs-tooltip-auto[x-placement^="bottom"]{padding:.4rem 0}.bs-tooltip-bottom .arrow,.bs-tooltip-auto[x-placement^="bottom"] .arrow{top:0}.bs-tooltip-bottom .arrow::before,.bs-tooltip-auto[x-placement^="bottom"] .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-left,.bs-tooltip-auto[x-placement^="left"]{padding:0 .4rem}.bs-tooltip-left .arrow,.bs-tooltip-auto[x-placement^="left"] .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-left .arrow::before,.bs-tooltip-auto[x-placement^="left"] .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::before,.popover .arrow::after{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-top,.bs-popover-auto[x-placement^="top"]{margin-bottom:.5rem}.bs-popover-top>.arrow,.bs-popover-auto[x-placement^="top"]>.arrow{bottom:calc((.5rem + 1px) * -1)}.bs-popover-top>.arrow::before,.bs-popover-auto[x-placement^="top"]>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,0.25)}.bs-popover-top>.arrow::after,.bs-popover-auto[x-placement^="top"]>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-right,.bs-popover-auto[x-placement^="right"]{margin-left:.5rem}.bs-popover-right>.arrow,.bs-popover-auto[x-placement^="right"]>.arrow{left:calc((.5rem + 1px) * -1);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-right>.arrow::before,.bs-popover-auto[x-placement^="right"]>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,0.25)}.bs-popover-right>.arrow::after,.bs-popover-auto[x-placement^="right"]>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-bottom,.bs-popover-auto[x-placement^="bottom"]{margin-top:.5rem}.bs-popover-bottom>.arrow,.bs-popover-auto[x-placement^="bottom"]>.arrow{top:calc((.5rem + 1px) * -1)}.bs-popover-bottom>.arrow::before,.bs-popover-auto[x-placement^="bottom"]>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,0.25)}.bs-popover-bottom>.arrow::after,.bs-popover-auto[x-placement^="bottom"]>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-bottom .popover-header::before,.bs-popover-auto[x-placement^="bottom"] .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-left,.bs-popover-auto[x-placement^="left"]{margin-right:.5rem}.bs-popover-left>.arrow,.bs-popover-auto[x-placement^="left"]>.arrow{right:calc((.5rem + 1px) * -1);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-left>.arrow::before,.bs-popover-auto[x-placement^="left"]>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,0.25)}.bs-popover-left>.arrow::after,.bs-popover-auto[x-placement^="left"]>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;backface-visibility:hidden;transition:transform .6s ease-in-out}@media (prefers-reduced-motion: reduce){.carousel-item{transition:none}}.carousel-item.active,.carousel-item-next,.carousel-item-prev{display:block}.carousel-item-next:not(.carousel-item-left),.active.carousel-item-right{transform:translateX(100%)}.carousel-item-prev:not(.carousel-item-right),.active.carousel-item-left{transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item.active,.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:0s .6s opacity}@media (prefers-reduced-motion: reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-prev,.carousel-control-next{position:absolute;top:0;bottom:0;z-index:1;display:flex;align-items:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity 0.15s ease}@media (prefers-reduced-motion: reduce){.carousel-control-prev,.carousel-control-next{transition:none}}.carousel-control-prev:hover,.carousel-control-prev:focus,.carousel-control-next:hover,.carousel-control-next:focus{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-prev-icon,.carousel-control-next-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50% / 100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:flex;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity 0.6s ease}@media (prefers-reduced-motion: reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@keyframes spinner-border{to{transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline !important}.align-top{vertical-align:top !important}.align-middle{vertical-align:middle !important}.align-bottom{vertical-align:bottom !important}.align-text-bottom{vertical-align:text-bottom !important}.align-text-top{vertical-align:text-top !important}.bg-primary{background-color:#0093f9 !important}a.bg-primary:hover,a.bg-primary:focus,button.bg-primary:hover,button.bg-primary:focus{background-color:#0075c6 !important}.bg-secondary{background-color:#B2DEFD !important}a.bg-secondary:hover,a.bg-secondary:focus,button.bg-secondary:hover,button.bg-secondary:focus{background-color:#80c9fc !important}.bg-success{background-color:#15CAB4 !important}a.bg-success:hover,a.bg-success:focus,button.bg-success:hover,button.bg-success:focus{background-color:#109c8b !important}.bg-info{background-color:#9665ed !important}a.bg-info:hover,a.bg-info:focus,button.bg-info:hover,button.bg-info:focus{background-color:#7737e8 !important}.bg-warning{background-color:#ffa516 !important}a.bg-warning:hover,a.bg-warning:focus,button.bg-warning:hover,button.bg-warning:focus{background-color:#e28b00 !important}.bg-danger{background-color:#ff4051 !important}a.bg-danger:hover,a.bg-danger:focus,button.bg-danger:hover,button.bg-danger:focus{background-color:#ff0d23 !important}.bg-light{background-color:#f8f9fa !important}a.bg-light:hover,a.bg-light:focus,button.bg-light:hover,button.bg-light:focus{background-color:#dae0e5 !important}.bg-dark{background-color:#343a40 !important}a.bg-dark:hover,a.bg-dark:focus,button.bg-dark:hover,button.bg-dark:focus{background-color:#1d2124 !important}.bg-white{background-color:#fff !important}.bg-transparent{background-color:transparent !important}.border{border:1px solid #dee2e6 !important}.border-top{border-top:1px solid #dee2e6 !important}.border-right{border-right:1px solid #dee2e6 !important}.border-bottom{border-bottom:1px solid #dee2e6 !important}.border-left{border-left:1px solid #dee2e6 !important}.border-0{border:0 !important}.border-top-0{border-top:0 !important}.border-right-0{border-right:0 !important}.border-bottom-0{border-bottom:0 !important}.border-left-0{border-left:0 !important}.border-primary{border-color:#0093f9 !important}.border-secondary{border-color:#B2DEFD !important}.border-success{border-color:#15CAB4 !important}.border-info{border-color:#9665ed !important}.border-warning{border-color:#ffa516 !important}.border-danger{border-color:#ff4051 !important}.border-light{border-color:#f8f9fa !important}.border-dark{border-color:#343a40 !important}.border-white{border-color:#fff !important}.rounded-sm{border-radius:.2rem !important}.rounded{border-radius:.25rem !important}.rounded-top{border-top-left-radius:.25rem !important;border-top-right-radius:.25rem !important}.rounded-right{border-top-right-radius:.25rem !important;border-bottom-right-radius:.25rem !important}.rounded-bottom{border-bottom-right-radius:.25rem !important;border-bottom-left-radius:.25rem !important}.rounded-left{border-top-left-radius:.25rem !important;border-bottom-left-radius:.25rem !important}.rounded-lg{border-radius:.3rem !important}.rounded-circle{border-radius:50% !important}.rounded-pill{border-radius:50rem !important}.rounded-0{border-radius:0 !important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none !important}.d-inline{display:inline !important}.d-inline-block{display:inline-block !important}.d-block{display:block !important}.d-table{display:table !important}.d-table-row{display:table-row !important}.d-table-cell{display:table-cell !important}.d-flex{display:flex !important}.d-inline-flex{display:inline-flex !important}@media (min-width: 576px){.d-sm-none{display:none !important}.d-sm-inline{display:inline !important}.d-sm-inline-block{display:inline-block !important}.d-sm-block{display:block !important}.d-sm-table{display:table !important}.d-sm-table-row{display:table-row !important}.d-sm-table-cell{display:table-cell !important}.d-sm-flex{display:flex !important}.d-sm-inline-flex{display:inline-flex !important}}@media (min-width: 768px){.d-md-none{display:none !important}.d-md-inline{display:inline !important}.d-md-inline-block{display:inline-block !important}.d-md-block{display:block !important}.d-md-table{display:table !important}.d-md-table-row{display:table-row !important}.d-md-table-cell{display:table-cell !important}.d-md-flex{display:flex !important}.d-md-inline-flex{display:inline-flex !important}}@media (min-width: 992px){.d-lg-none{display:none !important}.d-lg-inline{display:inline !important}.d-lg-inline-block{display:inline-block !important}.d-lg-block{display:block !important}.d-lg-table{display:table !important}.d-lg-table-row{display:table-row !important}.d-lg-table-cell{display:table-cell !important}.d-lg-flex{display:flex !important}.d-lg-inline-flex{display:inline-flex !important}}@media (min-width: 1200px){.d-xl-none{display:none !important}.d-xl-inline{display:inline !important}.d-xl-inline-block{display:inline-block !important}.d-xl-block{display:block !important}.d-xl-table{display:table !important}.d-xl-table-row{display:table-row !important}.d-xl-table-cell{display:table-cell !important}.d-xl-flex{display:flex !important}.d-xl-inline-flex{display:inline-flex !important}}@media print{.d-print-none{display:none !important}.d-print-inline{display:inline !important}.d-print-inline-block{display:inline-block !important}.d-print-block{display:block !important}.d-print-table{display:table !important}.d-print-table-row{display:table-row !important}.d-print-table-cell{display:table-cell !important}.d-print-flex{display:flex !important}.d-print-inline-flex{display:inline-flex !important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive iframe,.embed-responsive embed,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.85714%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{flex-direction:row !important}.flex-column{flex-direction:column !important}.flex-row-reverse{flex-direction:row-reverse !important}.flex-column-reverse{flex-direction:column-reverse !important}.flex-wrap{flex-wrap:wrap !important}.flex-nowrap{flex-wrap:nowrap !important}.flex-wrap-reverse{flex-wrap:wrap-reverse !important}.flex-fill{flex:1 1 auto !important}.flex-grow-0{flex-grow:0 !important}.flex-grow-1{flex-grow:1 !important}.flex-shrink-0{flex-shrink:0 !important}.flex-shrink-1{flex-shrink:1 !important}.justify-content-start{justify-content:flex-start !important}.justify-content-end{justify-content:flex-end !important}.justify-content-center{justify-content:center !important}.justify-content-between{justify-content:space-between !important}.justify-content-around{justify-content:space-around !important}.align-items-start{align-items:flex-start !important}.align-items-end{align-items:flex-end !important}.align-items-center{align-items:center !important}.align-items-baseline{align-items:baseline !important}.align-items-stretch{align-items:stretch !important}.align-content-start{align-content:flex-start !important}.align-content-end{align-content:flex-end !important}.align-content-center{align-content:center !important}.align-content-between{align-content:space-between !important}.align-content-around{align-content:space-around !important}.align-content-stretch{align-content:stretch !important}.align-self-auto{align-self:auto !important}.align-self-start{align-self:flex-start !important}.align-self-end{align-self:flex-end !important}.align-self-center{align-self:center !important}.align-self-baseline{align-self:baseline !important}.align-self-stretch{align-self:stretch !important}@media (min-width: 576px){.flex-sm-row{flex-direction:row !important}.flex-sm-column{flex-direction:column !important}.flex-sm-row-reverse{flex-direction:row-reverse !important}.flex-sm-column-reverse{flex-direction:column-reverse !important}.flex-sm-wrap{flex-wrap:wrap !important}.flex-sm-nowrap{flex-wrap:nowrap !important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse !important}.flex-sm-fill{flex:1 1 auto !important}.flex-sm-grow-0{flex-grow:0 !important}.flex-sm-grow-1{flex-grow:1 !important}.flex-sm-shrink-0{flex-shrink:0 !important}.flex-sm-shrink-1{flex-shrink:1 !important}.justify-content-sm-start{justify-content:flex-start !important}.justify-content-sm-end{justify-content:flex-end !important}.justify-content-sm-center{justify-content:center !important}.justify-content-sm-between{justify-content:space-between !important}.justify-content-sm-around{justify-content:space-around !important}.align-items-sm-start{align-items:flex-start !important}.align-items-sm-end{align-items:flex-end !important}.align-items-sm-center{align-items:center !important}.align-items-sm-baseline{align-items:baseline !important}.align-items-sm-stretch{align-items:stretch !important}.align-content-sm-start{align-content:flex-start !important}.align-content-sm-end{align-content:flex-end !important}.align-content-sm-center{align-content:center !important}.align-content-sm-between{align-content:space-between !important}.align-content-sm-around{align-content:space-around !important}.align-content-sm-stretch{align-content:stretch !important}.align-self-sm-auto{align-self:auto !important}.align-self-sm-start{align-self:flex-start !important}.align-self-sm-end{align-self:flex-end !important}.align-self-sm-center{align-self:center !important}.align-self-sm-baseline{align-self:baseline !important}.align-self-sm-stretch{align-self:stretch !important}}@media (min-width: 768px){.flex-md-row{flex-direction:row !important}.flex-md-column{flex-direction:column !important}.flex-md-row-reverse{flex-direction:row-reverse !important}.flex-md-column-reverse{flex-direction:column-reverse !important}.flex-md-wrap{flex-wrap:wrap !important}.flex-md-nowrap{flex-wrap:nowrap !important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse !important}.flex-md-fill{flex:1 1 auto !important}.flex-md-grow-0{flex-grow:0 !important}.flex-md-grow-1{flex-grow:1 !important}.flex-md-shrink-0{flex-shrink:0 !important}.flex-md-shrink-1{flex-shrink:1 !important}.justify-content-md-start{justify-content:flex-start !important}.justify-content-md-end{justify-content:flex-end !important}.justify-content-md-center{justify-content:center !important}.justify-content-md-between{justify-content:space-between !important}.justify-content-md-around{justify-content:space-around !important}.align-items-md-start{align-items:flex-start !important}.align-items-md-end{align-items:flex-end !important}.align-items-md-center{align-items:center !important}.align-items-md-baseline{align-items:baseline !important}.align-items-md-stretch{align-items:stretch !important}.align-content-md-start{align-content:flex-start !important}.align-content-md-end{align-content:flex-end !important}.align-content-md-center{align-content:center !important}.align-content-md-between{align-content:space-between !important}.align-content-md-around{align-content:space-around !important}.align-content-md-stretch{align-content:stretch !important}.align-self-md-auto{align-self:auto !important}.align-self-md-start{align-self:flex-start !important}.align-self-md-end{align-self:flex-end !important}.align-self-md-center{align-self:center !important}.align-self-md-baseline{align-self:baseline !important}.align-self-md-stretch{align-self:stretch !important}}@media (min-width: 992px){.flex-lg-row{flex-direction:row !important}.flex-lg-column{flex-direction:column !important}.flex-lg-row-reverse{flex-direction:row-reverse !important}.flex-lg-column-reverse{flex-direction:column-reverse !important}.flex-lg-wrap{flex-wrap:wrap !important}.flex-lg-nowrap{flex-wrap:nowrap !important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse !important}.flex-lg-fill{flex:1 1 auto !important}.flex-lg-grow-0{flex-grow:0 !important}.flex-lg-grow-1{flex-grow:1 !important}.flex-lg-shrink-0{flex-shrink:0 !important}.flex-lg-shrink-1{flex-shrink:1 !important}.justify-content-lg-start{justify-content:flex-start !important}.justify-content-lg-end{justify-content:flex-end !important}.justify-content-lg-center{justify-content:center !important}.justify-content-lg-between{justify-content:space-between !important}.justify-content-lg-around{justify-content:space-around !important}.align-items-lg-start{align-items:flex-start !important}.align-items-lg-end{align-items:flex-end !important}.align-items-lg-center{align-items:center !important}.align-items-lg-baseline{align-items:baseline !important}.align-items-lg-stretch{align-items:stretch !important}.align-content-lg-start{align-content:flex-start !important}.align-content-lg-end{align-content:flex-end !important}.align-content-lg-center{align-content:center !important}.align-content-lg-between{align-content:space-between !important}.align-content-lg-around{align-content:space-around !important}.align-content-lg-stretch{align-content:stretch !important}.align-self-lg-auto{align-self:auto !important}.align-self-lg-start{align-self:flex-start !important}.align-self-lg-end{align-self:flex-end !important}.align-self-lg-center{align-self:center !important}.align-self-lg-baseline{align-self:baseline !important}.align-self-lg-stretch{align-self:stretch !important}}@media (min-width: 1200px){.flex-xl-row{flex-direction:row !important}.flex-xl-column{flex-direction:column !important}.flex-xl-row-reverse{flex-direction:row-reverse !important}.flex-xl-column-reverse{flex-direction:column-reverse !important}.flex-xl-wrap{flex-wrap:wrap !important}.flex-xl-nowrap{flex-wrap:nowrap !important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse !important}.flex-xl-fill{flex:1 1 auto !important}.flex-xl-grow-0{flex-grow:0 !important}.flex-xl-grow-1{flex-grow:1 !important}.flex-xl-shrink-0{flex-shrink:0 !important}.flex-xl-shrink-1{flex-shrink:1 !important}.justify-content-xl-start{justify-content:flex-start !important}.justify-content-xl-end{justify-content:flex-end !important}.justify-content-xl-center{justify-content:center !important}.justify-content-xl-between{justify-content:space-between !important}.justify-content-xl-around{justify-content:space-around !important}.align-items-xl-start{align-items:flex-start !important}.align-items-xl-end{align-items:flex-end !important}.align-items-xl-center{align-items:center !important}.align-items-xl-baseline{align-items:baseline !important}.align-items-xl-stretch{align-items:stretch !important}.align-content-xl-start{align-content:flex-start !important}.align-content-xl-end{align-content:flex-end !important}.align-content-xl-center{align-content:center !important}.align-content-xl-between{align-content:space-between !important}.align-content-xl-around{align-content:space-around !important}.align-content-xl-stretch{align-content:stretch !important}.align-self-xl-auto{align-self:auto !important}.align-self-xl-start{align-self:flex-start !important}.align-self-xl-end{align-self:flex-end !important}.align-self-xl-center{align-self:center !important}.align-self-xl-baseline{align-self:baseline !important}.align-self-xl-stretch{align-self:stretch !important}}.float-left{float:left !important}.float-right{float:right !important}.float-none{float:none !important}@media (min-width: 576px){.float-sm-left{float:left !important}.float-sm-right{float:right !important}.float-sm-none{float:none !important}}@media (min-width: 768px){.float-md-left{float:left !important}.float-md-right{float:right !important}.float-md-none{float:none !important}}@media (min-width: 992px){.float-lg-left{float:left !important}.float-lg-right{float:right !important}.float-lg-none{float:none !important}}@media (min-width: 1200px){.float-xl-left{float:left !important}.float-xl-right{float:right !important}.float-xl-none{float:none !important}}.overflow-auto{overflow:auto !important}.overflow-hidden{overflow:hidden !important}.position-static{position:static !important}.position-relative{position:relative !important}.position-absolute{position:absolute !important}.position-fixed{position:fixed !important}.position-sticky{position:sticky !important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports (position: sticky){.sticky-top{position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 0.125rem 0.25rem rgba(0,0,0,0.075) !important}.shadow{box-shadow:0 0.5rem 1rem rgba(0,0,0,0.15) !important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,0.175) !important}.shadow-none{box-shadow:none !important}.w-25{width:25% !important}.w-50{width:50% !important}.w-75{width:75% !important}.w-100{width:100% !important}.w-auto{width:auto !important}.h-25{height:25% !important}.h-50{height:50% !important}.h-75{height:75% !important}.h-100{height:100% !important}.h-auto{height:auto !important}.mw-100{max-width:100% !important}.mh-100{max-height:100% !important}.min-vw-100{min-width:100vw !important}.min-vh-100{min-height:100vh !important}.vw-100{width:100vw !important}.vh-100{height:100vh !important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.m-0{margin:0 !important}.mt-0,.my-0{margin-top:0 !important}.mr-0,.mx-0{margin-right:0 !important}.mb-0,.my-0{margin-bottom:0 !important}.ml-0,.mx-0{margin-left:0 !important}.m-1{margin:.25rem !important}.mt-1,.my-1{margin-top:.25rem !important}.mr-1,.mx-1{margin-right:.25rem !important}.mb-1,.my-1{margin-bottom:.25rem !important}.ml-1,.mx-1{margin-left:.25rem !important}.m-2{margin:.5rem !important}.mt-2,.my-2{margin-top:.5rem !important}.mr-2,.mx-2{margin-right:.5rem !important}.mb-2,.my-2{margin-bottom:.5rem !important}.ml-2,.mx-2{margin-left:.5rem !important}.m-3{margin:1rem !important}.mt-3,.my-3{margin-top:1rem !important}.mr-3,.mx-3{margin-right:1rem !important}.mb-3,.my-3{margin-bottom:1rem !important}.ml-3,.mx-3{margin-left:1rem !important}.m-4{margin:1.5rem !important}.mt-4,.my-4{margin-top:1.5rem !important}.mr-4,.mx-4{margin-right:1.5rem !important}.mb-4,.my-4{margin-bottom:1.5rem !important}.ml-4,.mx-4{margin-left:1.5rem !important}.m-5{margin:3rem !important}.mt-5,.my-5{margin-top:3rem !important}.mr-5,.mx-5{margin-right:3rem !important}.mb-5,.my-5{margin-bottom:3rem !important}.ml-5,.mx-5{margin-left:3rem !important}.p-0{padding:0 !important}.pt-0,.py-0{padding-top:0 !important}.pr-0,.px-0{padding-right:0 !important}.pb-0,.py-0{padding-bottom:0 !important}.pl-0,.px-0{padding-left:0 !important}.p-1{padding:.25rem !important}.pt-1,.py-1{padding-top:.25rem !important}.pr-1,.px-1{padding-right:.25rem !important}.pb-1,.py-1{padding-bottom:.25rem !important}.pl-1,.px-1{padding-left:.25rem !important}.p-2{padding:.5rem !important}.pt-2,.py-2{padding-top:.5rem !important}.pr-2,.px-2{padding-right:.5rem !important}.pb-2,.py-2{padding-bottom:.5rem !important}.pl-2,.px-2{padding-left:.5rem !important}.p-3{padding:1rem !important}.pt-3,.py-3{padding-top:1rem !important}.pr-3,.px-3{padding-right:1rem !important}.pb-3,.py-3{padding-bottom:1rem !important}.pl-3,.px-3{padding-left:1rem !important}.p-4{padding:1.5rem !important}.pt-4,.py-4{padding-top:1.5rem !important}.pr-4,.px-4{padding-right:1.5rem !important}.pb-4,.py-4{padding-bottom:1.5rem !important}.pl-4,.px-4{padding-left:1.5rem !important}.p-5{padding:3rem !important}.pt-5,.py-5{padding-top:3rem !important}.pr-5,.px-5{padding-right:3rem !important}.pb-5,.py-5{padding-bottom:3rem !important}.pl-5,.px-5{padding-left:3rem !important}.m-n1{margin:-.25rem !important}.mt-n1,.my-n1{margin-top:-.25rem !important}.mr-n1,.mx-n1{margin-right:-.25rem !important}.mb-n1,.my-n1{margin-bottom:-.25rem !important}.ml-n1,.mx-n1{margin-left:-.25rem !important}.m-n2{margin:-.5rem !important}.mt-n2,.my-n2{margin-top:-.5rem !important}.mr-n2,.mx-n2{margin-right:-.5rem !important}.mb-n2,.my-n2{margin-bottom:-.5rem !important}.ml-n2,.mx-n2{margin-left:-.5rem !important}.m-n3{margin:-1rem !important}.mt-n3,.my-n3{margin-top:-1rem !important}.mr-n3,.mx-n3{margin-right:-1rem !important}.mb-n3,.my-n3{margin-bottom:-1rem !important}.ml-n3,.mx-n3{margin-left:-1rem !important}.m-n4{margin:-1.5rem !important}.mt-n4,.my-n4{margin-top:-1.5rem !important}.mr-n4,.mx-n4{margin-right:-1.5rem !important}.mb-n4,.my-n4{margin-bottom:-1.5rem !important}.ml-n4,.mx-n4{margin-left:-1.5rem !important}.m-n5{margin:-3rem !important}.mt-n5,.my-n5{margin-top:-3rem !important}.mr-n5,.mx-n5{margin-right:-3rem !important}.mb-n5,.my-n5{margin-bottom:-3rem !important}.ml-n5,.mx-n5{margin-left:-3rem !important}.m-auto{margin:auto !important}.mt-auto,.my-auto{margin-top:auto !important}.mr-auto,.mx-auto{margin-right:auto !important}.mb-auto,.my-auto{margin-bottom:auto !important}.ml-auto,.mx-auto{margin-left:auto !important}@media (min-width: 576px){.m-sm-0{margin:0 !important}.mt-sm-0,.my-sm-0{margin-top:0 !important}.mr-sm-0,.mx-sm-0{margin-right:0 !important}.mb-sm-0,.my-sm-0{margin-bottom:0 !important}.ml-sm-0,.mx-sm-0{margin-left:0 !important}.m-sm-1{margin:.25rem !important}.mt-sm-1,.my-sm-1{margin-top:.25rem !important}.mr-sm-1,.mx-sm-1{margin-right:.25rem !important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem !important}.ml-sm-1,.mx-sm-1{margin-left:.25rem !important}.m-sm-2{margin:.5rem !important}.mt-sm-2,.my-sm-2{margin-top:.5rem !important}.mr-sm-2,.mx-sm-2{margin-right:.5rem !important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem !important}.ml-sm-2,.mx-sm-2{margin-left:.5rem !important}.m-sm-3{margin:1rem !important}.mt-sm-3,.my-sm-3{margin-top:1rem !important}.mr-sm-3,.mx-sm-3{margin-right:1rem !important}.mb-sm-3,.my-sm-3{margin-bottom:1rem !important}.ml-sm-3,.mx-sm-3{margin-left:1rem !important}.m-sm-4{margin:1.5rem !important}.mt-sm-4,.my-sm-4{margin-top:1.5rem !important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem !important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem !important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem !important}.m-sm-5{margin:3rem !important}.mt-sm-5,.my-sm-5{margin-top:3rem !important}.mr-sm-5,.mx-sm-5{margin-right:3rem !important}.mb-sm-5,.my-sm-5{margin-bottom:3rem !important}.ml-sm-5,.mx-sm-5{margin-left:3rem !important}.p-sm-0{padding:0 !important}.pt-sm-0,.py-sm-0{padding-top:0 !important}.pr-sm-0,.px-sm-0{padding-right:0 !important}.pb-sm-0,.py-sm-0{padding-bottom:0 !important}.pl-sm-0,.px-sm-0{padding-left:0 !important}.p-sm-1{padding:.25rem !important}.pt-sm-1,.py-sm-1{padding-top:.25rem !important}.pr-sm-1,.px-sm-1{padding-right:.25rem !important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem !important}.pl-sm-1,.px-sm-1{padding-left:.25rem !important}.p-sm-2{padding:.5rem !important}.pt-sm-2,.py-sm-2{padding-top:.5rem !important}.pr-sm-2,.px-sm-2{padding-right:.5rem !important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem !important}.pl-sm-2,.px-sm-2{padding-left:.5rem !important}.p-sm-3{padding:1rem !important}.pt-sm-3,.py-sm-3{padding-top:1rem !important}.pr-sm-3,.px-sm-3{padding-right:1rem !important}.pb-sm-3,.py-sm-3{padding-bottom:1rem !important}.pl-sm-3,.px-sm-3{padding-left:1rem !important}.p-sm-4{padding:1.5rem !important}.pt-sm-4,.py-sm-4{padding-top:1.5rem !important}.pr-sm-4,.px-sm-4{padding-right:1.5rem !important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem !important}.pl-sm-4,.px-sm-4{padding-left:1.5rem !important}.p-sm-5{padding:3rem !important}.pt-sm-5,.py-sm-5{padding-top:3rem !important}.pr-sm-5,.px-sm-5{padding-right:3rem !important}.pb-sm-5,.py-sm-5{padding-bottom:3rem !important}.pl-sm-5,.px-sm-5{padding-left:3rem !important}.m-sm-n1{margin:-.25rem !important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem !important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem !important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem !important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem !important}.m-sm-n2{margin:-.5rem !important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem !important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem !important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem !important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem !important}.m-sm-n3{margin:-1rem !important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem !important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem !important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem !important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem !important}.m-sm-n4{margin:-1.5rem !important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem !important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem !important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem !important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem !important}.m-sm-n5{margin:-3rem !important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem !important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem !important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem !important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem !important}.m-sm-auto{margin:auto !important}.mt-sm-auto,.my-sm-auto{margin-top:auto !important}.mr-sm-auto,.mx-sm-auto{margin-right:auto !important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto !important}.ml-sm-auto,.mx-sm-auto{margin-left:auto !important}}@media (min-width: 768px){.m-md-0{margin:0 !important}.mt-md-0,.my-md-0{margin-top:0 !important}.mr-md-0,.mx-md-0{margin-right:0 !important}.mb-md-0,.my-md-0{margin-bottom:0 !important}.ml-md-0,.mx-md-0{margin-left:0 !important}.m-md-1{margin:.25rem !important}.mt-md-1,.my-md-1{margin-top:.25rem !important}.mr-md-1,.mx-md-1{margin-right:.25rem !important}.mb-md-1,.my-md-1{margin-bottom:.25rem !important}.ml-md-1,.mx-md-1{margin-left:.25rem !important}.m-md-2{margin:.5rem !important}.mt-md-2,.my-md-2{margin-top:.5rem !important}.mr-md-2,.mx-md-2{margin-right:.5rem !important}.mb-md-2,.my-md-2{margin-bottom:.5rem !important}.ml-md-2,.mx-md-2{margin-left:.5rem !important}.m-md-3{margin:1rem !important}.mt-md-3,.my-md-3{margin-top:1rem !important}.mr-md-3,.mx-md-3{margin-right:1rem !important}.mb-md-3,.my-md-3{margin-bottom:1rem !important}.ml-md-3,.mx-md-3{margin-left:1rem !important}.m-md-4{margin:1.5rem !important}.mt-md-4,.my-md-4{margin-top:1.5rem !important}.mr-md-4,.mx-md-4{margin-right:1.5rem !important}.mb-md-4,.my-md-4{margin-bottom:1.5rem !important}.ml-md-4,.mx-md-4{margin-left:1.5rem !important}.m-md-5{margin:3rem !important}.mt-md-5,.my-md-5{margin-top:3rem !important}.mr-md-5,.mx-md-5{margin-right:3rem !important}.mb-md-5,.my-md-5{margin-bottom:3rem !important}.ml-md-5,.mx-md-5{margin-left:3rem !important}.p-md-0{padding:0 !important}.pt-md-0,.py-md-0{padding-top:0 !important}.pr-md-0,.px-md-0{padding-right:0 !important}.pb-md-0,.py-md-0{padding-bottom:0 !important}.pl-md-0,.px-md-0{padding-left:0 !important}.p-md-1{padding:.25rem !important}.pt-md-1,.py-md-1{padding-top:.25rem !important}.pr-md-1,.px-md-1{padding-right:.25rem !important}.pb-md-1,.py-md-1{padding-bottom:.25rem !important}.pl-md-1,.px-md-1{padding-left:.25rem !important}.p-md-2{padding:.5rem !important}.pt-md-2,.py-md-2{padding-top:.5rem !important}.pr-md-2,.px-md-2{padding-right:.5rem !important}.pb-md-2,.py-md-2{padding-bottom:.5rem !important}.pl-md-2,.px-md-2{padding-left:.5rem !important}.p-md-3{padding:1rem !important}.pt-md-3,.py-md-3{padding-top:1rem !important}.pr-md-3,.px-md-3{padding-right:1rem !important}.pb-md-3,.py-md-3{padding-bottom:1rem !important}.pl-md-3,.px-md-3{padding-left:1rem !important}.p-md-4{padding:1.5rem !important}.pt-md-4,.py-md-4{padding-top:1.5rem !important}.pr-md-4,.px-md-4{padding-right:1.5rem !important}.pb-md-4,.py-md-4{padding-bottom:1.5rem !important}.pl-md-4,.px-md-4{padding-left:1.5rem !important}.p-md-5{padding:3rem !important}.pt-md-5,.py-md-5{padding-top:3rem !important}.pr-md-5,.px-md-5{padding-right:3rem !important}.pb-md-5,.py-md-5{padding-bottom:3rem !important}.pl-md-5,.px-md-5{padding-left:3rem !important}.m-md-n1{margin:-.25rem !important}.mt-md-n1,.my-md-n1{margin-top:-.25rem !important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem !important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem !important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem !important}.m-md-n2{margin:-.5rem !important}.mt-md-n2,.my-md-n2{margin-top:-.5rem !important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem !important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem !important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem !important}.m-md-n3{margin:-1rem !important}.mt-md-n3,.my-md-n3{margin-top:-1rem !important}.mr-md-n3,.mx-md-n3{margin-right:-1rem !important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem !important}.ml-md-n3,.mx-md-n3{margin-left:-1rem !important}.m-md-n4{margin:-1.5rem !important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem !important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem !important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem !important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem !important}.m-md-n5{margin:-3rem !important}.mt-md-n5,.my-md-n5{margin-top:-3rem !important}.mr-md-n5,.mx-md-n5{margin-right:-3rem !important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem !important}.ml-md-n5,.mx-md-n5{margin-left:-3rem !important}.m-md-auto{margin:auto !important}.mt-md-auto,.my-md-auto{margin-top:auto !important}.mr-md-auto,.mx-md-auto{margin-right:auto !important}.mb-md-auto,.my-md-auto{margin-bottom:auto !important}.ml-md-auto,.mx-md-auto{margin-left:auto !important}}@media (min-width: 992px){.m-lg-0{margin:0 !important}.mt-lg-0,.my-lg-0{margin-top:0 !important}.mr-lg-0,.mx-lg-0{margin-right:0 !important}.mb-lg-0,.my-lg-0{margin-bottom:0 !important}.ml-lg-0,.mx-lg-0{margin-left:0 !important}.m-lg-1{margin:.25rem !important}.mt-lg-1,.my-lg-1{margin-top:.25rem !important}.mr-lg-1,.mx-lg-1{margin-right:.25rem !important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem !important}.ml-lg-1,.mx-lg-1{margin-left:.25rem !important}.m-lg-2{margin:.5rem !important}.mt-lg-2,.my-lg-2{margin-top:.5rem !important}.mr-lg-2,.mx-lg-2{margin-right:.5rem !important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem !important}.ml-lg-2,.mx-lg-2{margin-left:.5rem !important}.m-lg-3{margin:1rem !important}.mt-lg-3,.my-lg-3{margin-top:1rem !important}.mr-lg-3,.mx-lg-3{margin-right:1rem !important}.mb-lg-3,.my-lg-3{margin-bottom:1rem !important}.ml-lg-3,.mx-lg-3{margin-left:1rem !important}.m-lg-4{margin:1.5rem !important}.mt-lg-4,.my-lg-4{margin-top:1.5rem !important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem !important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem !important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem !important}.m-lg-5{margin:3rem !important}.mt-lg-5,.my-lg-5{margin-top:3rem !important}.mr-lg-5,.mx-lg-5{margin-right:3rem !important}.mb-lg-5,.my-lg-5{margin-bottom:3rem !important}.ml-lg-5,.mx-lg-5{margin-left:3rem !important}.p-lg-0{padding:0 !important}.pt-lg-0,.py-lg-0{padding-top:0 !important}.pr-lg-0,.px-lg-0{padding-right:0 !important}.pb-lg-0,.py-lg-0{padding-bottom:0 !important}.pl-lg-0,.px-lg-0{padding-left:0 !important}.p-lg-1{padding:.25rem !important}.pt-lg-1,.py-lg-1{padding-top:.25rem !important}.pr-lg-1,.px-lg-1{padding-right:.25rem !important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem !important}.pl-lg-1,.px-lg-1{padding-left:.25rem !important}.p-lg-2{padding:.5rem !important}.pt-lg-2,.py-lg-2{padding-top:.5rem !important}.pr-lg-2,.px-lg-2{padding-right:.5rem !important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem !important}.pl-lg-2,.px-lg-2{padding-left:.5rem !important}.p-lg-3{padding:1rem !important}.pt-lg-3,.py-lg-3{padding-top:1rem !important}.pr-lg-3,.px-lg-3{padding-right:1rem !important}.pb-lg-3,.py-lg-3{padding-bottom:1rem !important}.pl-lg-3,.px-lg-3{padding-left:1rem !important}.p-lg-4{padding:1.5rem !important}.pt-lg-4,.py-lg-4{padding-top:1.5rem !important}.pr-lg-4,.px-lg-4{padding-right:1.5rem !important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem !important}.pl-lg-4,.px-lg-4{padding-left:1.5rem !important}.p-lg-5{padding:3rem !important}.pt-lg-5,.py-lg-5{padding-top:3rem !important}.pr-lg-5,.px-lg-5{padding-right:3rem !important}.pb-lg-5,.py-lg-5{padding-bottom:3rem !important}.pl-lg-5,.px-lg-5{padding-left:3rem !important}.m-lg-n1{margin:-.25rem !important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem !important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem !important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem !important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem !important}.m-lg-n2{margin:-.5rem !important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem !important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem !important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem !important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem !important}.m-lg-n3{margin:-1rem !important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem !important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem !important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem !important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem !important}.m-lg-n4{margin:-1.5rem !important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem !important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem !important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem !important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem !important}.m-lg-n5{margin:-3rem !important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem !important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem !important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem !important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem !important}.m-lg-auto{margin:auto !important}.mt-lg-auto,.my-lg-auto{margin-top:auto !important}.mr-lg-auto,.mx-lg-auto{margin-right:auto !important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto !important}.ml-lg-auto,.mx-lg-auto{margin-left:auto !important}}@media (min-width: 1200px){.m-xl-0{margin:0 !important}.mt-xl-0,.my-xl-0{margin-top:0 !important}.mr-xl-0,.mx-xl-0{margin-right:0 !important}.mb-xl-0,.my-xl-0{margin-bottom:0 !important}.ml-xl-0,.mx-xl-0{margin-left:0 !important}.m-xl-1{margin:.25rem !important}.mt-xl-1,.my-xl-1{margin-top:.25rem !important}.mr-xl-1,.mx-xl-1{margin-right:.25rem !important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem !important}.ml-xl-1,.mx-xl-1{margin-left:.25rem !important}.m-xl-2{margin:.5rem !important}.mt-xl-2,.my-xl-2{margin-top:.5rem !important}.mr-xl-2,.mx-xl-2{margin-right:.5rem !important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem !important}.ml-xl-2,.mx-xl-2{margin-left:.5rem !important}.m-xl-3{margin:1rem !important}.mt-xl-3,.my-xl-3{margin-top:1rem !important}.mr-xl-3,.mx-xl-3{margin-right:1rem !important}.mb-xl-3,.my-xl-3{margin-bottom:1rem !important}.ml-xl-3,.mx-xl-3{margin-left:1rem !important}.m-xl-4{margin:1.5rem !important}.mt-xl-4,.my-xl-4{margin-top:1.5rem !important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem !important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem !important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem !important}.m-xl-5{margin:3rem !important}.mt-xl-5,.my-xl-5{margin-top:3rem !important}.mr-xl-5,.mx-xl-5{margin-right:3rem !important}.mb-xl-5,.my-xl-5{margin-bottom:3rem !important}.ml-xl-5,.mx-xl-5{margin-left:3rem !important}.p-xl-0{padding:0 !important}.pt-xl-0,.py-xl-0{padding-top:0 !important}.pr-xl-0,.px-xl-0{padding-right:0 !important}.pb-xl-0,.py-xl-0{padding-bottom:0 !important}.pl-xl-0,.px-xl-0{padding-left:0 !important}.p-xl-1{padding:.25rem !important}.pt-xl-1,.py-xl-1{padding-top:.25rem !important}.pr-xl-1,.px-xl-1{padding-right:.25rem !important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem !important}.pl-xl-1,.px-xl-1{padding-left:.25rem !important}.p-xl-2{padding:.5rem !important}.pt-xl-2,.py-xl-2{padding-top:.5rem !important}.pr-xl-2,.px-xl-2{padding-right:.5rem !important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem !important}.pl-xl-2,.px-xl-2{padding-left:.5rem !important}.p-xl-3{padding:1rem !important}.pt-xl-3,.py-xl-3{padding-top:1rem !important}.pr-xl-3,.px-xl-3{padding-right:1rem !important}.pb-xl-3,.py-xl-3{padding-bottom:1rem !important}.pl-xl-3,.px-xl-3{padding-left:1rem !important}.p-xl-4{padding:1.5rem !important}.pt-xl-4,.py-xl-4{padding-top:1.5rem !important}.pr-xl-4,.px-xl-4{padding-right:1.5rem !important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem !important}.pl-xl-4,.px-xl-4{padding-left:1.5rem !important}.p-xl-5{padding:3rem !important}.pt-xl-5,.py-xl-5{padding-top:3rem !important}.pr-xl-5,.px-xl-5{padding-right:3rem !important}.pb-xl-5,.py-xl-5{padding-bottom:3rem !important}.pl-xl-5,.px-xl-5{padding-left:3rem !important}.m-xl-n1{margin:-.25rem !important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem !important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem !important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem !important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem !important}.m-xl-n2{margin:-.5rem !important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem !important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem !important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem !important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem !important}.m-xl-n3{margin:-1rem !important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem !important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem !important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem !important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem !important}.m-xl-n4{margin:-1.5rem !important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem !important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem !important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem !important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem !important}.m-xl-n5{margin:-3rem !important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem !important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem !important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem !important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem !important}.m-xl-auto{margin:auto !important}.mt-xl-auto,.my-xl-auto{margin-top:auto !important}.mr-xl-auto,.mx-xl-auto{margin-right:auto !important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto !important}.ml-xl-auto,.mx-xl-auto{margin-left:auto !important}}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace !important}.text-justify{text-align:justify !important}.text-wrap{white-space:normal !important}.text-nowrap{white-space:nowrap !important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left !important}.text-right{text-align:right !important}.text-center{text-align:center !important}@media (min-width: 576px){.text-sm-left{text-align:left !important}.text-sm-right{text-align:right !important}.text-sm-center{text-align:center !important}}@media (min-width: 768px){.text-md-left{text-align:left !important}.text-md-right{text-align:right !important}.text-md-center{text-align:center !important}}@media (min-width: 992px){.text-lg-left{text-align:left !important}.text-lg-right{text-align:right !important}.text-lg-center{text-align:center !important}}@media (min-width: 1200px){.text-xl-left{text-align:left !important}.text-xl-right{text-align:right !important}.text-xl-center{text-align:center !important}}.text-lowercase{text-transform:lowercase !important}.text-uppercase{text-transform:uppercase !important}.text-capitalize{text-transform:capitalize !important}.font-weight-light{font-weight:300 !important}.font-weight-lighter{font-weight:lighter !important}.font-weight-normal{font-weight:400 !important}.font-weight-bold{font-weight:700 !important}.font-weight-bolder{font-weight:bolder !important}.font-italic{font-style:italic !important}.text-white{color:#fff !important}.text-primary{color:#0093f9 !important}a.text-primary:hover,a.text-primary:focus{color:#0066ad !important}.text-secondary{color:#B2DEFD !important}a.text-secondary:hover,a.text-secondary:focus{color:#67befb !important}.text-success{color:#15CAB4 !important}a.text-success:hover,a.text-success:focus{color:#0e8576 !important}.text-info{color:#9665ed !important}a.text-info:hover,a.text-info:focus{color:#6721e5 !important}.text-warning{color:#ffa516 !important}a.text-warning:hover,a.text-warning:focus{color:#c97b00 !important}.text-danger{color:#ff4051 !important}a.text-danger:hover,a.text-danger:focus{color:#f30016 !important}.text-light{color:#f8f9fa !important}a.text-light:hover,a.text-light:focus{color:#cbd3da !important}.text-dark{color:#343a40 !important}a.text-dark:hover,a.text-dark:focus{color:#121416 !important}.text-body{color:#212529 !important}.text-muted{color:#6c757d !important}.text-black-50{color:rgba(0,0,0,0.5) !important}.text-white-50{color:rgba(255,255,255,0.5) !important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none !important}.text-break{word-break:break-word !important;overflow-wrap:break-word !important}.text-reset{color:inherit !important}.visible{visibility:visible !important}.invisible{visibility:hidden !important}@media print{*,*::before,*::after{text-shadow:none !important;box-shadow:none !important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap !important}pre,blockquote{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px !important}.container{min-width:992px !important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse !important}.table td,.table th{background-color:#fff !important}.table-bordered th,.table-bordered td{border:1px solid #dee2e6 !important}.table-dark{color:inherit}.table-dark th,.table-dark td,.table-dark thead th,.table-dark tbody+tbody{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}}html{font-size:.8rem}@media (min-width: 576px){html{font-size:.9rem}}@media (min-width: 768px){html{font-size:.9rem}}@media (min-width: 992px){html{font-size:1rem}}
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<zk>
+ <borderlayout vflex="1" viewModel="@id('vm') @init('hu.user.trainmanager.viewmodel.DatabaseViewModel')">
+ <north>
+ <grid hflex="1" vflex="min">
+ <auxhead>
+ <auxheader colspan="2" >
+ <hbox align="center" pack="center">
+ <label hflex="1" value="Schema" style="text-align:center"/>
+ <combobox hflex="2" model="@load(vm.schemas)" selectedItem="@bind(vm.schema)"
+ onChange="@command('changeSchema')" placeholder="choose schema"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </hbox>
+ </auxheader>
+ <auxheader colspan="2" >
+ <hbox align="center" pack="center">
+ <label hflex="1" value="Table"/>
+ <combobox hflex="2" model="@load(vm.tables)" selectedItem="@bind(vm.table)"
+ onChange="@command('changeTable')" placeholder="choose table"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ <button iconSclass="z-icon-refresh" onClick="@command('refreshTables')"/>
+ </hbox>
+ </auxheader>
+ <auxheader colspan="2">
+ <hbox align="center" pack="center">
+ <label value="Offset"/>
+ <combobox hflex="1" model="@load(vm.offsetValues)" selectedItem="@bind(vm.offset)"
+ onChange="@command('changeOffset')" placeholder="select" style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </hbox>
+ </auxheader>
+ <auxheader colspan="2">
+ <hbox align="center" pack="center">
+ <label value="Limit" style="text-align:center"/>
+ <combobox hflex="1" model="@load(vm.limitValues)" selectedItem="@bind(vm.limit)"
+ onChange="@command('changeLimit')" onClick="@command('changeLimit')"
+ placeholder="select" style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </hbox>
+ </auxheader>
+ <auxheader colspan="2">
+ <hbox align="center" pack="center">
+ <label hflex="1" value="Records" style="text-align:center"/>
+ <textbox hflex="1" value="@load(vm.recordNumber)" style="text-align:center"/>
+ </hbox>
+ </auxheader>
+ <auxheader>
+ <checkbox label="type" checked="@save(vm.showTypeChecked)"/>
+ <checkbox label="split" checked="@save(vm.showSplitChecked)"/>
+ </auxheader>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="3"/>
+ <column hflex="1"/>
+ <column hflex="3"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="2"/>
+ </columns>
+ </grid>
+ </north>
+ <center>
+ <listbox model="@load(vm.records)" vflex="1" style="overflow:auto;" sizedByContent="true"
+ span="true">
+ <listhead children="@load(vm.fieldNames)" sizable="true">
+ <template name="children" var="item">
+ <!-- <listheader align="center" hflex="1">-->
+ <listheader hflex="min" align="center">
+ <label value="@load(item)" style="white-space: normal;" multiline="true"/>
+ </listheader>
+ </template>
+ </listhead>
+ <template name="model" var="line">
+ <listitem children="@bind(line)">
+ <template name="children" var="cell">
+ <listcell label="@load(cell)" style="text-align: center"/>
+ </template>
+ </listitem>
+ </template>
+ <listfoot children="@load(vm.fieldTypes)" visible="@load(vm.showTypeChecked)">
+ <template name="children" var="item">
+ <listfooter align="center">
+ <label value="@load(item)" style="white-space: normal;" multiline="true"/>
+ </listfooter>
+ </template>
+ </listfoot>
+ </listbox>
+ </center>
+ <south>
+ <hbox hflex="1" vflex="min" align="center" pack="center" visible="@load(vm.showSplitChecked)">
+ <hbox hflex="min" align="center" pack="center">
+ <label hflex="1" value="Fraud type field name"/>
+ <combobox hflex="1" model="@load(vm.fraudTypeFieldNames)"
+ selectedItem="@bind(vm.fraudTypeFieldName)"
+ onChange="@command('refreshFraudCaseNumbers')"
+ placeholder="select field" style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)" style="text-align:center"/>
+ </template>
+ </combobox>
+ </hbox>
+ <panel hflex="1" border="normal">
+ <panelchildren>
+ <label value="Positive fraud number"/>
+ <slider curpos="@save(vm.desiredPositiveFraudNumber)"
+ maxpos="@load(vm.positiveCaseNumber)"/>
+ </panelchildren>
+ </panel>
+ <!-- <hbox hflex="2" align="center" pack="center">-->
+ <!-- <label hflex="2" value="Positive fraud number"/>-->
+ <!-- <slider hflex="1" curpos="@save(vm.desiredPositiveFraudNumber)"/>-->
+ <!-- </hbox>-->
+ <panel hflex="1" border="normal">
+ <panelchildren>
+ <label value="Negative fraud number"/>
+ <slider curpos="@save(vm.desiredNegativeFraudNumber)"
+ maxpos="@load(vm.negativeCaseNumber)"/>
+ </panelchildren>
+
+ </panel>
+ <!-- <hbox hflex="2" align="right" pack="center">-->
+ <!-- <label hflex="2" value="Negative fraud number"/>-->
+ <!-- <slider hflex="1" curpos="@save(vm.desiredNegativeFraudNumber)"/>-->
+ <!-- </hbox>-->
+ <cell hflex="1" align="center">
+ <checkbox label="add date and time suffix" checked="@save(vm.enableAddDateTimeSuffix)"/>
+ </cell>
+<!-- <hbox hflex="1">-->
+ <button label="SPLIT DATABASE" onClick="@command('splitDatabase')"
+ disabled="@load(empty vm.table or empty vm.fraudTypeFieldName or
+ vm.desiredPositiveFraudNumber eq 0 or vm.desiredNegativeFraudNumber eq 0)"/>
+<!-- </hbox>-->
+ </hbox>
+ </south>
+ </borderlayout>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<window></window>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?link href="http://netdna.bootstrapcdn.com/font-awesome/4.0.1/css/font-awesome.css" rel="stylesheet"?>
+<?link rel="stylesheet" href="webjars/bootstrap/4.6.0/css/bootstrap.min.css"?>
+<?link rel="stylesheet" href="~./css/zk-admin-bootstrap-theme.css"?>
+<?link rel="stylesheet" type="text/css" href="http://yui.yahooapis.com/3.18.1/build/cssreset/cssreset-min.css"?>
+
+<zk>
+ <borderlayout height="100%" apply="hu.user.trainmanager.controller.TrainTaskController">
+ <center vflex="1" autoscroll="true">
+ <listbox id="progressListbox" mold="paging" pageSize="4" sizedByContent="true" span="true">
+ <auxhead>
+ <auxheader colspan="10" align="center">
+ <label value="TRAIN TASKS"/>
+ <button id="trainTasksRefreshButton" iconSclass="z-icon-refresh"/>
+ </auxheader>
+ </auxhead>
+ <listhead sizable="true">
+ <listheader align="center">Id</listheader>
+ <listheader tooltiptext="Train Task Name" align="center">Name</listheader>
+ <listheader align="center">Encoding</listheader>
+ <listheader align="center">
+ <label value="Feature Engineering"/>
+ </listheader>
+ <listheader align="center">Sampling process</listheader>
+ <listheader align="center">Fit process</listheader>
+ <listheader align="center">Estimator persisting</listheader>
+ <listheader align="center">Metrics persisting</listheader>
+ <listheader align="center">State</listheader>
+ <listheader align="center">Select</listheader>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell label="${each.id}"/>
+ <listcell label="${each.trainTaskName}"/>
+ <listcell label="${each.encodingState}"/>
+ <listcell label="${each.featureEngineeringState}"/>
+ <listcell label="${each.samplingProcessState}"/>
+ <listcell label="${each.fitProcessState}"/>
+ <listcell label="${each.estimatorPersistingState}"/>
+ <listcell label="${each.metricsPersistingState}"/>
+ <listcell label="${each.trainTaskState}"
+ iconSclass="${each.trainTaskState eq 'IN_PROGRESS' ? 'z-icon-spinner z-icon-spin' : 'null'}"/>
+ <listcell>
+ <button disabled="${each.addButtonDisabled}" iconSclass="z-icon-plus"
+ forward="onClick=progressListbox.onAddButtonClick"/>
+ </listcell>
+ </listitem>
+ </template>
+ </listbox>
+ </center>
+ <south vflex="min">
+ <grid>
+ <columns>
+ <column hflex="3" align="center"/>
+ <column hflex="1" align="center"/>
+ <column hflex="3" align="center"/>
+ <column hflex="1" align="center"/>
+ <column hflex="1" align="center"/>
+ <column hflex="1" align="center"/>
+ <column hflex="3" align="center"/>
+ <column hflex="1" align="center"/>
+ <column hflex="2" align="center"/>
+ <column hflex="2" align="center"/>
+ <column hflex="2" align="center"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label value="All Train Tasks"/>
+ </cell>
+ <cell align="center">
+ <textbox hflex="1" id="allTrainTaskNuber" style="text-align:center"/>
+ </cell>
+ <cell>
+ <label hflex="2" value="Finished Train Tasks"/>
+ </cell>
+ <cell align="center">
+ <textbox hflex="1" id="finishedTrainTaskNumber" style="text-align:center"/>
+ </cell>
+ <cell>
+ <label hflex="1" value="To-do"/>
+ </cell>
+ <cell align="center">
+ <textbox hflex="1" id="tobeDoneTrainTaskNumber" style="text-align:center"/>
+ </cell>
+ <cell>
+ <label hflex="1" value="Currently in Progress"/>
+ </cell>
+ <cell align="center">
+ <textbox hflex="1" id="currentLyInProgressTrainTaskId" style="text-align:center"/>
+ </cell>
+ <cell>
+ <label hflex="1" value="CPU Core"/>
+ </cell>
+ <cell align="center">
+ <combobox hflex="1" id="cpuCoreComboBox" style="text-align:center"/>
+ </cell>
+ <cell>
+ <button hflex="1" id="startButton" label="START" iconSclass="z-icon-check"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </south>
+ </borderlayout>
+
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" ?>
+<hbox viewModel="@id('vm') @init('hu.user.trainmanager.viewmodel.InputTrainParameterViewModel')"
+ validationMessages="@id('vmsgs')" sizedByContent="true">
+ <!-- <vbox hflex="1" sizedByContent="true">-->
+ <vbox hflex="1">
+ <grid hflex="3" span="true" sizedByContent="true">
+ <auxhead>
+ <auxheader colspan="2" label="Database" align="center"/>
+ <auxheader colspan="2" label="Specific properties" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label hflex="1" value="Schema name"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.schemas)" selectedItem="@bind(vm.schema)"
+ placeholder="choose schema" style="text-align:center"
+ onChange="@command('changeSchema')">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ <cell>
+ <label hflex="1" value="Timebase field name"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.eligibleTimeBaseFields)" placeholder="choose time base"
+ selectedItem="@bind(vm.chosenTimeBaseField)" style="text-align:center"
+ onChange="@command('changeTimeBaseField')">
+ <template name="model">
+ <comboitem label="@load(each)" style="text-align:center"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label hflex="1" value="Table name"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.tables)" selectedItem="@bind(vm.table)"
+ placeholder="choose table" style="text-align:center"
+ onChange="@command('changeTable')">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ <cell>
+ <label value="Transaction identifier"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.columnsContainOnlyDistinctValues)"
+ placeholder="choose field"
+ selectedItem="@bind(vm.uniqueFieldName)" style="text-align:center"
+ onChange="@command('changeUniqueField')">
+ <template name="model">
+ <comboitem label="@load(each)" style="text-align:center"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ <grid>
+ <auxhead>
+ <auxheader colspan="4" label="Feature Engineering" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="2"/>
+ <column hflex="1"/>
+ <column hflex="2"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label hflex="1" value="Enable engineering"/>
+ </cell>
+ <cell align="center">
+ <checkbox hflex="1"
+ disabled="@load(not vm.allEncodingParametersSet or not empty vm.encodingErrorMessage)"
+ checked="@bind(vm.enabledFeatureEngineering)"></checkbox>
+ </cell>
+ <cell>
+ <label hflex="1" value="Feature field name"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" disabled="@load(!vm.enabledFeatureEngineering)" placeholder="Choose field"
+ model="@load(vm.fieldCandidatesForFeatureEngineering)" style="text-align:center"
+ selectedItem="@bind(vm.chosenFeatureField)"
+ onChange="@command('checkIfSelectedFeatureFieldExist')">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ <row visible="@load(not empty vm.featureEngineeringErrorMessage)">
+ <cell/>
+ <cell/>
+ <cell>
+ <label value="Error message"/>
+ </cell>
+ <cell>
+ <textbox value="@load(vm.featureEngineeringErrorMessage)" style="text-align:center"/>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label hflex="2" value="Retrospective day"/>
+ </cell>
+ <cell>
+ <hbox align="center" pack="center">
+ <spinner hflex="min"
+ disabled="@load(!vm.enabledFeatureEngineering or empty vm.chosenFeatureField)"
+ constraint="no negative,no empty,no zero, min 1"
+ value="@bind(vm.retroSpectiveDay)" style="text-align:center"/>
+ <button disabled="@load(!vm.enabledFeatureEngineering or empty vm.chosenFeatureField)"
+ onClick="@command('addDayToIntervals')" iconSclass="z-icon-plus-square"
+ tooltiptext="Add Retrospective day">
+ </button>
+ <button hflex="min" disabled="@load(empty vm.presentedIntervals)" iconSclass="z-icon-trash" onClick="@command('clearPresentedIntervals')"/>
+ </hbox>
+ </cell>
+ <cell>
+ <label hflex="1" value="Intervals"/>
+ </cell>
+ <cell>
+ <textbox hflex="1" disabled="@load(!vm.enabledFeatureEngineering)"
+ placeholder="selected days" value="@load(vm.presentedIntervals)"
+ style="text-align:center"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </vbox>
+ <vbox align="center" hflex="1" vflex="true">
+ <grid>
+ <auxhead>
+ <auxheader label="Encoding parameters" colspan="3" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <textbox value="@load(vm.currentField)" hflex="1" readonly="true"
+ placeholder="field name" style="text-align:center"/>
+ </cell>
+ <cell>
+ <combobox model="@load(vm.currentApplicableEncodingTypes)"
+ selectedItem="@bind(vm.currentEncodingType)"
+ hflex="1" placeholder="choose type"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ <cell>
+ <hbox align="center" pack="center">
+ <button hflex="1" disabled="@load(vm.disabledNextEncodingTypeButton)"
+ onClick="@command('saveCurrentFieldAndEncodingTypeAndGetNextField')"
+ label="Next">
+ </button>
+ <button hflex="1" disabled="@load(vm.disabledFinishEncodingTypeButton)"
+ onClick="@command('saveLastFieldAndEncodingType')" label="Finish">
+ </button>
+ </hbox>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ <hbox visible="@load(not empty vm.encodingErrorMessage)" hflex="1" align="center">
+ <label value="Error message" style="text-align:center"/>
+ <textbox hflex="3" value="@load(vm.encodingErrorMessage)" style="text-align:center"/>
+ <button hflex="min" iconSclass="z-icon-repeat" onClick="@command('clearEncodingParametersAfterError')"/>
+ </hbox>
+
+
+ <grid hflex="1" sizedByContent="true" span="true">
+ <auxhead>
+ <auxheader label="Pipeline parameters" colspan="4" align="center"/>
+ </auxhead>
+ <columns>
+ <column hflex="1"/>
+ <column hflex="2"/>
+ <column hflex="1"/>
+ <column hflex="2"/>
+ </columns>
+ <rows>
+ <row>
+ <cell>
+ <label value="Feature Selector"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.featureSelectors)" value="@bind(vm.featureSelectorName)"
+ placeholder="choose selector" style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ <cell>
+ <label value="Sampler"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.samplers)" value="@bind(vm.samplerName)" placeholder="choose sampler"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label value="Scaler"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.scalers)" value="@bind(vm.scalerName)" placeholder="choose scaler"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ <cell>
+ <label value="Model"/>
+ </cell>
+ <cell>
+ <combobox hflex="1" model="@load(vm.models)" value="@bind(vm.modelName)" placeholder="choose model"
+ style="text-align:center">
+ <template name="model">
+ <comboitem label="@load(each)"/>
+ </template>
+ </combobox>
+ </cell>
+ </row>
+ <row>
+ <cell>
+ <label value="Test size (%)"/>
+ </cell>
+ <cell>
+ <spinner hflex="1" step="1" constraint="no empty,min 1 max 95" value="@bind(vm.testSizePercent)"
+ placeholder="set test size" style="text-align:center"/>
+ </cell>
+ <cell>
+ <label value="Task name" hflex="1"/>
+ </cell>
+ <cell>
+ <textbox hflex="1" id="text_box" tooltiptext="Train Task name" placeholder="type name"
+ constraint="no empty: Please enter Train Task name"
+ value="@bind(vm.trainTaskName) @validator(vm.trainTaskNameValidator)"
+ style="text-align:center"/>
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ <grid visible="@load(not empty vmsgs[text_box])">
+ <columns>
+ <column hflex="1"/>
+ <column hflex="1"/>
+ </columns>
+ <rows>
+ <row align="center">
+ <cell>
+ <label hflex="1" value="Error message"/>
+ </cell>
+ <cell align="center">
+ <textbox hflex="1" value="@load(vmsgs[text_box])" style="text-align:center"/>
+ </cell>
+ </row>
+ </rows>
+
+ </grid>
+ <hbox align="center" pack="center">
+ <button disabled="@load(vm.allEncodingParametersSet eq false or vm.modelName eq null or vm.scalerName eq null or
+ vm.samplerName eq null or vm.testSizePercent lt 1 or vm.trainTaskName eq null or vm.chosenTimeBaseField eq null or
+ vm.uniqueFieldName eq null or not empty vmsgs[text_box]) or not empty vm.featureEngineeringErrorMessage)"
+ iconSclass="z-icon-save" tooltiptext="Save Train Task" onClick="@command('saveTrainTask')"/>
+ <button iconSclass="z-icon-trash-o" tooltiptext="Clear Train Task parameters"
+ disabled="@load(vm.schema eq null)" onClick="@command('clearAllEncodingParameters')"/>
+ </hbox>
+ </vbox>
+</hbox>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<zk>
+ <tabbox height="100%" mold="accordion"
+ viewModel="@id('vm') @init('hu.user.trainmanager.viewmodel.MetricsViewModel')">
+ <tabs align="center">
+ <tab label="Show Metrics"/>
+ <tab label="Save Metrics"/>
+ </tabs>
+ <tabpanels>
+ <tabpanel>
+ <!-- <listbox height="400px" model="@load(vm.estimatorMetrics)" sizedByContent="true">-->
+ <listbox vflex="true" model="@load(vm.metricsCsvContentDto.lines)" sizedByContent="true">
+ <auxhead>
+ <auxheader colspan="24" align="center">
+ <hbox align="center">
+ <button hflex="min" iconSclass="z-icon-refresh" onClick="@command('refreshMetrics')"/>
+ <label value="METRIC TYPES"/>
+ </hbox>
+ </auxheader>
+ </auxhead>
+ <listhead children="@load(vm.metricsCsvContentDto.headerElements)" sizable="true">
+ <template name="children" var="item">
+ <listheader align="center">
+ <label value="@load(item)" style="white-space: normal;" multiline="true"/>
+ </listheader>
+ </template>
+ </listhead>
+ <template name="model" var="line">
+ <listitem children="@bind(line)">
+ <template name="children" var="cell">
+ <listcell label="@load(cell)" style="text-align: center"/>
+ </template>
+ </listitem>
+ </template>
+ </listbox>
+ </tabpanel>
+ <tabpanel>
+ <borderlayout vflex="1">
+ <north>
+ <hbox align="center" sizedByContent="true">
+ <label value="Full Path: "/>
+ <textbox hflex="true" value="@load(vm.fullPath)"/>
+ </hbox>
+ </north>
+ <west width="30%">
+ <listbox model="@load(vm.directoryProperties)" vflex="true">
+ <listhead>
+ <listheader label="Directory"/>
+ </listhead>
+ </listbox>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.getDisplayName())"
+ onClick="@command('changeDirectory',param=each.directory)"/>
+ </listitem>
+ </template>
+ </west>
+ <center>
+ <listbox model="@load(vm.fileProperties)" mold="paging" pageSize="4" vflex="true">
+ <listhead>
+ <listheader hflex="4" label="File name" align="center"/>
+ <listheader hflex="1" label="OPERATION" align="center"/>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.fileName)"/>
+ <listcell>
+ <hbox align="center">
+ <button iconSclass="z-icon-trash-o"
+ onClick="@command('deleteSelectedCsvFile',param=each.file)"/>
+ <button iconSclass="z-icon-download"
+ onClick="@command('downloadSelectedCsvFile',param=each.file)"/>
+ </hbox>
+ </listcell>
+ </listitem>
+ </template>
+ </listbox>
+ </center>
+ <east width="20%">
+
+ <grid width="100%" vflex="true">
+ <columns>
+ <column hflex="1" label="CSV FILE" align="center"/>
+ </columns>
+ <rows>
+ <row>
+ <textbox placeholder="type Train Task file name"
+ constraint="no empty: Please enter Train Task csv file name"
+ value="@bind(vm.trainTaskCsvFileName)"
+ />
+ </row>
+ <row>
+ <cell align="center">
+ <label value="date and time suffix"/>
+ <checkbox disabled="@load(empty vm.trainTaskCsvFileName)"
+ checked="@save(vm.enabledTrainTaskCsvDateTimeSuffix)"/>
+ <button disabled="@load(empty vm.trainTaskCsvFileName)"
+ tooltiptext="Create Train Task csv file" iconSclass="z-icon-save"
+ onClick="@command('createTrainTaskCsvFile')"/>
+ </cell>
+ </row>
+ <row>
+ <textbox placeholder="type Metrics csv file name"
+ constraint="no empty: Please enter Metrics csv file name"
+ value="@bind(vm.metricsCsvFileName)"/>
+ </row>
+ <row>
+ <cell align="center">
+ <!-- <hbox align="center" pack="center">-->
+ <label hflex="2" value="date and time suffix"/>
+ <checkbox hflex="1" disabled="@load(empty vm.metricsCsvFileName)"
+ checked="@save(vm.enabledMetricsCsvDateTimeSuffix)"/>
+ <button disabled="@load(empty vm.metricsCsvFileName)"
+ tooltiptext="Create Metrics csv file" iconSclass="z-icon-save"
+ onClick="@command('createMetricsCsvFile')"/>
+ <!-- </hbox>-->
+
+ </cell>
+ </row>
+ </rows>
+ </grid>
+ </east>
+ </borderlayout>
+ </tabpanel>
+ </tabpanels>
+ </tabbox>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?taglib uri="http://www.zkoss.org/dsp/web/core" prefix="c" ?>
+<zk>
+ <window
+ title="SAVE METRICS"
+ id="inputCpuParameterFormId" mode="modal"
+ minimizable="false" border="normal" position="center,center" closable="true"
+ onClose="@command('cancel')"
+ action="show: slideDown;hide: slideUp" sclass="row-title">
+ <caption sclass="caption" label="SAVE METRICS to csv file"/>
+ <borderlayout height="450px">
+ <north>
+ <hbox align="center">
+ <label value="Full Path: "/>
+ <textbox value="@load(vm.fullPath)"/>
+ </hbox>
+ </north>
+ <west width="30%">
+ <listbox model="@load(vm.directoryProperties)" vflex="true">
+ <listhead>
+ <listheader label="Directory"/>
+ </listhead>
+ </listbox>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.getDisplayName())"
+ onClick="@command('changeDirectory',param=each.directory)"/>
+ </listitem>
+ </template>
+ </west>
+ <center>
+ <listbox model="@load(vm.fileProperties)" mold="paging" pageSize="4" height="100%"
+ selectedIndex="@bind(vm.selectedFileNameIndex)">
+ <listhead>
+ <listheader label="File name" align="center"/>
+ </listhead>
+ <template name="model">
+ <listitem>
+ <listcell label="@load(each.fileName)"/>
+ </listitem>
+ </template>
+ </listbox>
+ </center>
+ <east>
+ <vbox align="center">
+ <label value="File name"/>
+ <textbox value="@save(vm.csvFilename)" onChange="@command()"/>
+ <button disabled="@load(vm.saveButtonDisabled)" label="SAVE"/>
+ </vbox>
+ </east>
+ </borderlayout>
+ </window>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" ?>
+<zk>
+ <div viewModel="@id('vm') @init('storedEncodingAndFeatureEngineeringParameters')"/>
+ <vbox sizedByContent="true">
+ <listbox model="@init(vm.storedEncodingAndFeatureEngineeringParameters)"
+ selectedItem="@bind(vm.selectedStoredEncodingAndFeatureEngineeringParameter)"
+ onSelect="@command('setDisplayedEncodingAndFeatureEngineeringParameters')">
+ <auxhead>
+ <auxheader label="Stored Encoding and Feature Engineering Parameters" colspan="5" align="center"/>
+ </auxhead>
+ <listheader>
+ <listhead colspan="5" label="Stored Encoding and Feature Engineering Parameters"/>
+ </listheader>
+ <listhead>
+ <listheader hflex="1">Id</listheader>
+ <listheader hflex="4">Schema name</listheader>
+ <listheader hflex="4">Table name</listheader>
+ <listheader hflex="4">Timebase field name</listheader>
+ <listheader hflex="2">Select</listheader>
+ </listhead>
+ <template name="model">
+ <row>
+ <textbox value="@bind(each.id)"/>
+ <textbox value="@bind(each.schemaName)"/>
+ <textbox value="@bind(each.tableName)"/>
+ <textbox value="@bind(each.timeBaseFieldName)"/>
+ <button label="Select"
+ onClick="@command('setEncodingAndFeatureEngineeringId',id=each.id"/>
+ </row>
+ </template>
+ </listbox>
+ <hbox>
+ <grid model="vm.displayedEncodingParameters">
+ <auxhead>
+ <auxheader label="Encoding Parameters" colspan="2" align="center"/>
+ </auxhead>
+ <template name="model">
+ <row>
+ <textbox value="@load(each.fieldName"/>
+ <textbox value="@load(each.encodingType"/>
+ </row>
+ </template>
+ </grid>
+ <grid>
+ <auxhead>
+ <auxheader label="Feature Engineering Parameters" colspan="2" align="center"/>
+ </auxhead>
+ <columns>
+ <column>Chosen field name</column>
+ <column>intervals</column>
+ </columns>
+ <rows>
+ <row>
+ <textbox value="@load(displayedChosenFeatureField"/>
+ <textbox value="@load(displayedRetroSpectiveIntervals)"/>
+ </row>
+ </rows>
+ </grid>
+ </hbox>
+ </vbox>
+</zk>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<?taglib uri="http://www.zkoss.org/dsp/web/core" prefix="c" ?>
+<?page title="Train Manager"?>
+<zk>
+ <window border="normal" height="100%" width="100%" style="background:#f7f7f7;"
+ title="Train Manager">
+
+ <tabbox vflex="1">
+ <tabs>
+ <tab label="Database"/>
+ <tab label="Train task"/>
+ <tab label="Fit progress"/>
+ <tab label="Metrics"/>
+ </tabs>
+ <tabpanels>
+ <tabpanel>
+ <include src="~./zul/database.zul"/>
+ </tabpanel>
+ <tabpanel>
+ <include src="~./zul/inputTrainParameter.zul"/>
+ </tabpanel>
+ <tabpanel>
+ <include src="~./zul/fit.zul"/>
+ </tabpanel>
+ <tabpanel>
+ <include src="~./zul/metrics.zul"/>
+ </tabpanel>
+ </tabpanels>
+ </tabbox>
+ </window>
+</zk>
\ No newline at end of file
--- /dev/null
+package hu.user.trainmanager;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class TrainManagerApplicationTests {
+
+ @Test
+ void contextLoads() {
+ }
+
+}
--- /dev/null
+#version: "3.9"
+
+services:
+# core:
+# build:
+# context: ./Core
+# dockerfile: Dockerfile
+# environment:
+# - SPRING_PROFILES_ACTIVE=dockerized
+# - SPRING_RABBITMQ_HOST=rabbitmq
+# restart: always
+# ports:
+# - 8081:8081
+# networks:
+# - backend
+# depends_on:
+# rabbitmq:
+# condition: service_healthy
+
+# core-manager:
+# build:
+# context: ./CoreManager
+# dockerfile: Dockerfile
+# environment:
+# - SPRING_PROFILES_ACTIVE=dockerized
+# - SPRING_RABBITMQ_HOST=rabbitmq
+# restart: always
+# ports:
+# - 8082:8082
+# networks:
+# - backend
+# depends_on:
+# rabbitmq:
+# condition: service_healthy
+
+# train-manager:
+# build:
+# context: ./TrainManager
+# dockerfile: Dockerfile
+# environment:
+# - SPRING_PROFILES_ACTIVE=dockerized
+# - SPRING_RABBITMQ_HOST=rabbitmq
+# restart: always
+# ports:
+# - 8084:8084
+# networks:
+# - backend
+# depends_on:
+# rabbitmq:
+# condition: service_healthy
+
+ estimator:
+# build:
+# context: ./Estimator
+# dockerfile: Dockerfile
+ image: tomi70/estimator:2.0
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/log/estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+# build:
+# context: ./Train
+# dockerfile: Dockerfile
+ image: tomi70/train:2.0
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/log/train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ # restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+ build:
+ context: ./Core
+ dockerfile: Dockerfile
+# image: tomi70/core:2.0
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+ build:
+ context: ./CoreManager
+ dockerfile: Dockerfile
+# image: tomi70/core_manager:1.0
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ volumes:
+ - ~/log/core_manager:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+ build:
+ context: ./TrainManager
+ dockerfile: Dockerfile
+# image: tomi70/train_manager:1.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+ build:
+ context: ./Estimator
+ dockerfile: Dockerfile
+# image: tomi70/estimator:2.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/opt/app/log/Estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+ build:
+ context: ./Train
+ dockerfile: Dockerfile
+# image: tomi70/train:3.1.1
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/opt/app/log/Train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+# build:
+# context: ./Core
+# dockerfile: Dockerfile
+ image: tomi70/core:2.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+# build:
+# context: ./CoreManager
+# dockerfile: Dockerfile
+ image: tomi70/core_manager:1.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ volumes:
+ - ~/log/core_manager:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+# build:
+# context: ./TrainManager
+# dockerfile: Dockerfile
+ image: tomi70/train_manager:1.2
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+# build:
+# context: ./Estimator
+# dockerfile: Dockerfile
+ image: tomi70/estimator:2.1.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/opt/app/log/Estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+# build:
+# context: ./Train
+# dockerfile: Dockerfile
+ image: tomi70/train:3.1.2
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/opt/app/log/Train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+ # build:
+ # context: ./Core
+ # dockerfile: Dockerfile
+ image: tomi70/core:2.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+ # build:
+ # context: ./CoreManager
+ # dockerfile: Dockerfile
+ image: tomi70/core_manager:1.1.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ volumes:
+ - ~/log/core_manager:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+ # build:
+ # context: ./TrainManager
+ # dockerfile: Dockerfile
+ image: tomi70/train_manager:1.2.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+ # build:
+ # context: ./Estimator
+ # dockerfile: Dockerfile
+ image: tomi70/estimator:2.1.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/opt/app/log/Estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+ # build:
+ # context: ./Train
+ # dockerfile: Dockerfile
+ image: tomi70/train:3.1.2
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/opt/app/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/opt/app/log/Train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+# build:
+# context: ./Core
+# dockerfile: Dockerfile
+ image: tomi70/core:2.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/Core.log"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+# build:
+# context: ./CoreManager
+# dockerfile: Dockerfile
+ image: tomi70/core_manager:1.1.2
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/CoreManager.log"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+# build:
+# context: ./TrainManager
+# dockerfile: Dockerfile
+ image: tomi70/train_manager:1.2.3
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/TrainManager.log"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+# build:
+# context: ./Estimator
+# dockerfile: Dockerfile
+ image: tomi70/estimator:2.1.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/log/estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+# build:
+# context: ./Train
+# dockerfile: Dockerfile
+ image: tomi70/train:3.1.2
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/log/train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ # restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+# build:
+# context: ./Core
+# dockerfile: Dockerfile
+ image: tomi70/core:2.0
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/Core.log"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+# build:
+# context: ./CoreManager
+# dockerfile: Dockerfile
+ image: tomi70/core_manager:1.0
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/CoreManager.log"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+# build:
+# context: ./TrainManager
+# dockerfile: Dockerfile
+ image: tomi70/train_manager:1.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/TrainManager.log"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+# build:
+# context: ./Estimator
+# dockerfile: Dockerfile
+ image: tomi70/estimator:2.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/log/estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+# build:
+# context: ./Train
+# dockerfile: Dockerfile
+ image: tomi70/train:3.1.1
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/log/train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ # restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
--- /dev/null
+#version: "3.9"
+
+services:
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3307:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ - ./scripts/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+ environment:
+ - MYSQL_USER=fraud
+ - MYSQL_PASSWORD=password
+ - MYSQL_ROOT_PASSWORD=pwd
+
+networks:
+ backend:
+volumes:
+ db_data:
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>hu.user</groupId>
+ <artifactId>CreditCardFraudDetect</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+ <modules>
+ <module>TrainManager</module>
+ <module>CoreManager</module>
+ <module>Core</module>
+ <module>CommonDto</module>
+ <module>CoreCli</module>
+ <module>TrainCli</module>
+ </modules>
+</project>
\ No newline at end of file
--- /dev/null
+#version: "3.9"
+
+services:
+ core:
+# build:
+# context: ./Core
+# dockerfile: Dockerfile
+ image: tomi70/core:2.1
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/common_fraud?serverTimezone=UTC",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "create",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/Core.log"
+ }'
+ restart: always
+ ports:
+ - 8081:8081
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ core-manager:
+# build:
+# context: ./CoreManager
+# dockerfile: Dockerfile
+ image: tomi70/core_manager:1.1.2
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "validate",
+ "home.directory" : "/opt/app",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/CoreManager.log"
+ }'
+
+ restart: always
+ ports:
+ - 8082:8082
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train-manager:
+# build:
+# context: ./TrainManager
+# dockerfile: Dockerfile
+ image: tomi70/train_manager:1.2.3
+ environment:
+ - SPRING_PROFILES_ACTIVE=dockerized
+ - SPRING_RABBITMQ_HOST=rabbitmq
+ - 'SPRING_APPLICATION_JSON={
+ "spring.datasource.url" : "jdbc:mysql://mysql_db:3306/$common_fraud?useSSL=false",
+ "spring.datasource.username" : "root",
+ "spring.datasource.password" : "pwd",
+ "spring.jpa.hibernate.ddl-auto" : "update",
+ "home.directory" : "/opt/app",
+ "evaluation.csv.directory":"/opt/app/evaluation/csv",
+ "logging.level.root" : "info",
+ "logging.file" : "/opt/app/TrainManager.log"
+ }'
+ restart: on-failure
+ ports:
+ - 8084:8084
+ networks:
+ - backend
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ estimator:
+# build:
+# context: ./Estimator
+# dockerfile: Dockerfile
+ image: tomi70/estimator:2.1.1
+ restart: on-failure
+ ports:
+ - 8083:8083
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - ESTIMATOR_LOG_LEVEL=DEBUG
+ - ESTIMATOR_LOG_FILE=/log/estimator.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+ train:
+# build:
+# context: ./Train
+# dockerfile: Dockerfile
+ image: tomi70/train:3.1.2
+ restart: on-failure
+ ports:
+ - 8085:8085
+ networks:
+ - backend
+ volumes:
+ - ~/log:/log
+ environment:
+ - RABBITMQ_URL=rabbitmq
+ - MYSQL_DATABASE_URL=mysql_db
+ - MYSQL_USER=root
+ - MYSQL_ROOT_PASSWORD=pwd
+ - TRAIN_LOG_LEVEL=DEBUG
+ - TRAIN_LOG_FILE=/log/train.log
+ depends_on:
+ rabbitmq:
+ condition: service_healthy
+
+
+ rabbitmq:
+ container_name: rabbitmq
+ image: "rabbitmq:3-management"
+ command: rabbitmq-server
+ ports:
+ - 5672:5672
+ - 15672:15672
+ # restart: always
+ networks:
+ - backend
+ healthcheck:
+ test: [ "CMD", "rabbitmqctl", "status" ]
+ interval: 5s
+ timeout: 20s
+ retries: 5
+
+ mysql_db:
+ image: mysql:8.0.28
+ command: mysqld --default-authentication-plugin=mysql_native_password
+ restart: always
+ ports:
+ - 3306:3306
+ security_opt:
+ - seccomp:unconfined
+ networks:
+ - backend
+ volumes:
+ - db_data:/var/lib/mysql
+ environment:
+ - MYSQL_ROOT_PASSWORD=pwd
+
+
+networks:
+ backend:
+volumes:
+ db_data:
+
https://git.useribm.hu / mki-fd.git / commitdiff
