diff --git a/.DS_Store b/.DS_Store index 58f3f36..e6462ef 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/docker-compose.yml b/docker-compose.yml index 4b382e7..11c149d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,6 @@ services: restart: unless-stopped ports: - "9192:9192" - environment: - - DB_PASSWORD=Neu2026! - - HUMBEE_PASSWORD=Neu2026! - - AZURE_EMBEDDING_KEY=7QT99mzRAZEYTH9aETqPfNZ15637VSpe6POp9y3C1gCGbhg85dMHJQQJ99CFACPV0roXJ3w3AAAAACOGvNH3 - - OPENAI_API_MISTRAL_KEY=7QT99mzRAZEYTH9aETqPfNZ15637VSpe6POp9y3C1gCGbhg85dMHJQQJ99CFACPV0roXJ3w3AAAAACOGvNH3 + + diff --git a/src/.DS_Store b/src/.DS_Store index 47db383..57b5178 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/main/.DS_Store b/src/main/.DS_Store index b6521a1..cdc034c 100644 Binary files a/src/main/.DS_Store and b/src/main/.DS_Store differ diff --git a/src/main/java/com/homme/demo/controller/HumbeeController.java b/src/main/java/com/homme/demo/controller/HumbeeController.java index 97f7be2..6dfe2c8 100644 --- a/src/main/java/com/homme/demo/controller/HumbeeController.java +++ b/src/main/java/com/homme/demo/controller/HumbeeController.java @@ -7,26 +7,24 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; import com.homme.demo.dto.ProcessedWordFileResponseDto; import com.homme.demo.service.HumbeeService; -@RestController("humbee") +@RestController +@RequestMapping("/knowledge-base") public class HumbeeController { @Autowired private HumbeeService humbeeService; - - - - @GetMapping("/word-links") - public List getWordLinks() throws Exception { + @GetMapping("/build") + public List buildKnowledgeBase() throws Exception { - return humbeeService.getWordLinks(); - + return humbeeService.buildKnowledgeBase(); } diff --git a/src/main/java/com/homme/demo/controller/SearchController.java b/src/main/java/com/homme/demo/controller/SearchController.java index cac39b0..393cb33 100644 --- a/src/main/java/com/homme/demo/controller/SearchController.java +++ b/src/main/java/com/homme/demo/controller/SearchController.java @@ -3,12 +3,10 @@ package com.homme.demo.controller; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.homme.demo.dto.AskResponse; -import com.homme.demo.input.AskRequest; import com.homme.demo.service.RagService; @RestController @@ -19,9 +17,9 @@ public class SearchController { @PostMapping("/ask") - public AskResponse ask(@RequestBody AskRequest request) { + public AskResponse ask(@RequestParam String question) { - String question = (request == null) ? null : request.getQuestion(); + if (question == null || question.isBlank()) { return AskResponse.builder() diff --git a/src/main/java/com/homme/demo/input/AskRequest.java b/src/main/java/com/homme/demo/input/AskRequest.java deleted file mode 100644 index 6052b67..0000000 --- a/src/main/java/com/homme/demo/input/AskRequest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.homme.demo.input; - -import lombok.Getter; -import lombok.Setter; - -/** - * Request body for the chatbot endpoint: { "question": "..." }. - */ -@Setter -@Getter -public class AskRequest { - - private String question; - -} diff --git a/src/main/java/com/homme/demo/service/DocumentChunckService.java b/src/main/java/com/homme/demo/service/DocumentChunckService.java index e22ae59..1315709 100644 --- a/src/main/java/com/homme/demo/service/DocumentChunckService.java +++ b/src/main/java/com/homme/demo/service/DocumentChunckService.java @@ -16,8 +16,8 @@ public class DocumentChunckService { if(text == null || text.isBlank()) return chunks; - int chunkSize = 6000; - int overlab = 200; + int chunkSize = 1200; + int overlab = 150; int start = 0; while(start < text.length()) { diff --git a/src/main/java/com/homme/demo/service/HumbeeService.java b/src/main/java/com/homme/demo/service/HumbeeService.java index e6279e3..b979cb6 100644 --- a/src/main/java/com/homme/demo/service/HumbeeService.java +++ b/src/main/java/com/homme/demo/service/HumbeeService.java @@ -73,13 +73,13 @@ public class HumbeeService { .build(); } - public List getWordLinks() throws Exception{ + public List buildKnowledgeBase() throws Exception{ login(); String json = getDiroctoryJson(); List links = extractWordLinks(json); - - + int numFiles = 0; + int numFilesResult =0; ExecutorService pool = Executors.newFixedThreadPool(6); try { @@ -87,7 +87,7 @@ public class HumbeeService { for(String link :links) { - + numFiles++; futures.add(pool.submit(() -> processOneFile(link))); } @@ -95,6 +95,7 @@ public class HumbeeService { for(Future f: futures) { + numFilesResult++; result.add(f.get()); } return result; @@ -104,6 +105,7 @@ public class HumbeeService { Thread.currentThread().interrupt(); throw new RuntimeException("Fehler bei paralleler Dateiverarbeitung: " + e.getMessage()); } finally { + System.out.println("$$$$$$$$ ="+links.size()+" numFiles = "+numFiles +" numFilesResult= "+ numFilesResult); pool.shutdown(); } @@ -152,7 +154,6 @@ public class HumbeeService { public void login() { String loginPath = "/account/login"; - sessionCookies.clear(); Map requestBody = new HashMap<>(); requestBody.put("email", email); @@ -165,7 +166,6 @@ public class HumbeeService { .bodyValue(requestBody) .exchangeToMono(res -> Mono.just(res)) .block(); - System.out.println("response = "+response); if(response == null) { throw new RuntimeException("Anmeldung fehlgeschlagen: Keine Antwort vom Server."); @@ -213,7 +213,7 @@ public class HumbeeService { - } + } public List extractWordLinks(String json) throws Exception{ List links = new ArrayList<>(); @@ -231,7 +231,8 @@ public class HumbeeService { for(JsonNode action: quickActions) { String id = action.path("id").asText(); String href = action.path("href").asText(); - if("download".equals(id) && (href.contains(".docx?") || href.contains(".doc?"))) { + + if("download".equals(id) && (href.contains(".docx?") || href.contains(".doc?") || href.contains(".txt"))) { links.add("https://cloud.humbee.de"+href); } @@ -278,6 +279,11 @@ public class HumbeeService { } } + else if(downloadHref.contains(".txt")){ + + return decodeBytes(fileBytes); + + } throw new RuntimeException("Nicht unterstützer Dateityp"); } @@ -287,6 +293,26 @@ public class HumbeeService { throw new RuntimeException("Nicht angeledet. Bitte zuerst einloggen."); } } + + public String decodeBytes(byte[] bytes) { + if (bytes == null || bytes.length == 0) return ""; + + + if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF) + return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8); + if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE) + return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE); + if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF) + return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE); + + int nulls = 0; + for (int i = 1; i < Math.min(bytes.length, 400); i += 2) + if (bytes[i] == 0) nulls++; + if (nulls > 50) + return new String(bytes, java.nio.charset.StandardCharsets.UTF_16LE); + + return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); + } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f3aa5c4..51a85ae 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -3,7 +3,7 @@ spring.application.name=Homme # Load secrets from .env (classpath) so ${DB_PASSWORD}, ${HUMBEE_PASSWORD}, # ${AZURE_EMBEDDING_KEY}, ${OPENAI_API_MISTRAL_KEY} below resolve from it. # "optional:" => app still starts if .env is absent (e.g. secrets come from real env vars). -#spring.config.import=optional:classpath:/.env[.properties] +spring.config.import=optional:classpath:/.env[.properties] server.port =9192