Human-Library-Endpunkte
This commit is contained in:
@@ -16,3 +16,6 @@ HUMBEE_PASSWORD=
|
|||||||
|
|
||||||
# Optional extra JVM flags (e.g. -Xmx512m)
|
# Optional extra JVM flags (e.g. -Xmx512m)
|
||||||
JAVA_TOOL_OPTIONS=
|
JAVA_TOOL_OPTIONS=
|
||||||
|
|
||||||
|
# --- Admin-Token: schuetzt die Freigabe-Endpunkte (/contributions/pending, /approve) ---
|
||||||
|
HOMME_ADMIN_TOKEN=
|
||||||
|
|||||||
@@ -0,0 +1,273 @@
|
|||||||
|
# HÖMMA – Änderungsplan (Backend an Plugin anpassen)
|
||||||
|
|
||||||
|
خطة التعديل لمطابقة الباك-إند مع إضافة **HÖMMA Human Library**.
|
||||||
|
الفكرة: نضيف **Controller جديد واحد** يوفّر النقاط الثلاث تماماً كما يتوقّعها الـ plugin،
|
||||||
|
ويستدعي الخدمات الموجودة أصلاً. لا نلمس الـ Controllers الشغّالة.
|
||||||
|
|
||||||
|
الطريقة المختارة: **تعديل الباك-إند** (وليس الـ plugin).
|
||||||
|
النقاط الثلاث النهائية التي ستضعها في ووردبريس:
|
||||||
|
|
||||||
|
| خانة ووردبريس | الرابط النهائي (مثال Azure) |
|
||||||
|
|---|---|
|
||||||
|
| Speichern-Endpunkt | `https://<azure-url>/human-library` |
|
||||||
|
| Transkriptions-Endpunkt | `https://<azure-url>/transcribe` |
|
||||||
|
| Extraktions-Endpunkt | `https://<azure-url>/extract` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) pom.xml — إضافة دعم PDF
|
||||||
|
|
||||||
|
الـ plugin يسمح برفع Word **و PDF**، لكن المشروع يعالج `.docx` فقط.
|
||||||
|
أضف مكتبة PDFBox داخل `<dependencies>`:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.pdfbox</groupId>
|
||||||
|
<artifactId>pdfbox</artifactId>
|
||||||
|
<version>2.0.31</version>
|
||||||
|
</dependency>
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) WordExportService.java — تفعيل قراءة PDF
|
||||||
|
|
||||||
|
الملف: `src/main/java/com/homme/demo/service/WordExportService.java`
|
||||||
|
|
||||||
|
داخل دالة `leseDatei(MultipartFile file)`، **قبل** سطر `return knowledgeIngestService.decodeBytes(bytes);`
|
||||||
|
أضف فرع PDF:
|
||||||
|
|
||||||
|
```java
|
||||||
|
if (name.endsWith(".pdf")) {
|
||||||
|
try (org.apache.pdfbox.pdmodel.PDDocument doc =
|
||||||
|
org.apache.pdfbox.pdmodel.PDDocument.load(bytes)) {
|
||||||
|
return new org.apache.pdfbox.text.PDFTextStripper().getText(doc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
بذلك: `.docx` عبر POI، `.pdf` عبر PDFBox، والباقي يبقى كما هو.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3) DTO جديد — طلب الحفظ (JSON)
|
||||||
|
|
||||||
|
ملف جديد: `src/main/java/com/homme/demo/dto/HumanLibraryRequest.java`
|
||||||
|
|
||||||
|
الـ plugin يرسل JSON بهذه الحقول: `source, name, gender, city, text, consent`.
|
||||||
|
|
||||||
|
```java
|
||||||
|
package com.homme.demo.dto;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class HumanLibraryRequest {
|
||||||
|
private String source;
|
||||||
|
private String name;
|
||||||
|
private String gender;
|
||||||
|
private String city;
|
||||||
|
private String text;
|
||||||
|
private boolean consent;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) DTO جديد — الرد الموحّد `{ "text": "..." }`
|
||||||
|
|
||||||
|
ملف جديد: `src/main/java/com/homme/demo/dto/TextResponse.java`
|
||||||
|
|
||||||
|
الـ plugin يتوقّع من الاستخراج والتفريغ رداً بصيغة `{"text":"..."}`.
|
||||||
|
|
||||||
|
```java
|
||||||
|
package com.homme.demo.dto;
|
||||||
|
|
||||||
|
public record TextResponse(String text) {
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Controller جديد — النقاط الثلاث للـ plugin
|
||||||
|
|
||||||
|
ملف جديد: `src/main/java/com/homme/demo/controller/HumanLibraryController.java`
|
||||||
|
|
||||||
|
هذا هو قلب التعديل. يوفّر `/extract`, `/transcribe`, `/human-library`
|
||||||
|
بنفس أسماء الحقول والصيغ التي يتوقّعها الـ plugin، ويستدعي الخدمات الموجودة.
|
||||||
|
|
||||||
|
```java
|
||||||
|
package com.homme.demo.controller;
|
||||||
|
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
|
import com.homme.demo.dto.HumanLibraryRequest;
|
||||||
|
import com.homme.demo.dto.TextResponse;
|
||||||
|
import com.homme.demo.dto.UploadWordFileResponseDto;
|
||||||
|
import com.homme.demo.service.AzureSpeechService;
|
||||||
|
import com.homme.demo.service.KnowledgeIngestService;
|
||||||
|
import com.homme.demo.service.WordExportService;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
public class HumanLibraryController {
|
||||||
|
|
||||||
|
@Autowired private WordExportService wordExportService;
|
||||||
|
@Autowired private AzureSpeechService azureSpeechService;
|
||||||
|
@Autowired private KnowledgeIngestService knowledgeIngestService;
|
||||||
|
|
||||||
|
// (1) Extraktion: Word/PDF -> { "text": "..." }
|
||||||
|
@PostMapping(value = "/extract", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||||
|
public ResponseEntity<TextResponse> extract(@RequestParam("file") MultipartFile file) throws Exception {
|
||||||
|
if (file == null || file.isEmpty()) {
|
||||||
|
return ResponseEntity.badRequest().body(new TextResponse(""));
|
||||||
|
}
|
||||||
|
String text = wordExportService.leseDatei(file);
|
||||||
|
return ResponseEntity.ok(new TextResponse(text == null ? "" : text));
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) Transkription: audio -> { "text": "..." }
|
||||||
|
@PostMapping(value = "/transcribe", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||||
|
public ResponseEntity<TextResponse> transcribe(@RequestParam("audio") MultipartFile audio) throws Exception {
|
||||||
|
if (audio == null || audio.isEmpty()) {
|
||||||
|
return ResponseEntity.badRequest().body(new TextResponse(""));
|
||||||
|
}
|
||||||
|
String text = azureSpeechService.transcribe(audio.getBytes(), audio.getOriginalFilename());
|
||||||
|
return ResponseEntity.ok(new TextResponse(text == null ? "" : text));
|
||||||
|
}
|
||||||
|
|
||||||
|
// (3) Speichern: JSON { source, name, gender, city, text, consent } -> 201
|
||||||
|
@PostMapping(value = "/human-library", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public ResponseEntity<?> saveContribution(@RequestBody HumanLibraryRequest req) {
|
||||||
|
try {
|
||||||
|
if (!req.isConsent()) {
|
||||||
|
return ResponseEntity.badRequest().body("Einwilligung (consent) erforderlich.");
|
||||||
|
}
|
||||||
|
if (req.getText() == null || req.getText().isBlank()) {
|
||||||
|
return ResponseEntity.badRequest().body("Text erforderlich.");
|
||||||
|
}
|
||||||
|
|
||||||
|
String title = (req.getName() == null || req.getName().isBlank())
|
||||||
|
? "Anonymous" : req.getName().trim().replace(" ", "_");
|
||||||
|
String city = (req.getCity() == null || req.getCity().isBlank())
|
||||||
|
? "Dortmund" : req.getCity();
|
||||||
|
|
||||||
|
// Text -> DOCX -> Humbee-Upload -> RAG-Verarbeitung (wie save-new-interview)
|
||||||
|
byte[] docx = wordExportService.createInterviewDocx(req.getText());
|
||||||
|
String fileName = "%s_%s_%s.docx".formatted(LocalDate.now(), title, city);
|
||||||
|
|
||||||
|
UploadWordFileResponseDto upload = knowledgeIngestService.uploadWordFile(fileName, docx);
|
||||||
|
if (!upload.isSuccess()) {
|
||||||
|
return ResponseEntity.status(HttpStatus.BAD_GATEWAY)
|
||||||
|
.body("Upload fehlgeschlagen: " + upload.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
knowledgeIngestService.processOneFile(upload.getLink());
|
||||||
|
return ResponseEntity.status(HttpStatus.CREATED).body("{\"status\":\"ok\"}");
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
|
||||||
|
.body("Fehler beim Speichern: " + e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
> ملاحظة: هذا يعيد استخدام نفس منطق `save-new-interview` الموجود.
|
||||||
|
> إذا أردت لاحقاً مرحلة **مراجعة تحريرية (ausstehend)** قبل RAG، نضيف حالة/جدول قبل استدعاء `processOneFile`.
|
||||||
|
> حقلا `gender` و `source` يُستقبلان لكن لا يُخزَّنان حالياً — أضِفهما للـ entity لاحقاً إن لزم.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6) CORS — (اختياري، موصى به)
|
||||||
|
|
||||||
|
الملف: `src/main/java/com/homme/demo/config/WebConfig.java`
|
||||||
|
حالياً مفتوح للكل (`allowedOriginPatterns("*")`) — يعمل.
|
||||||
|
للحصر على نطاقك فقط (كما نصح صاحب العمل) استبدله بـ:
|
||||||
|
|
||||||
|
```java
|
||||||
|
registry.addMapping("/**")
|
||||||
|
.allowedOrigins("https://hmmahumanlibraryruhr1314.live-website.com")
|
||||||
|
.allowedMethods("GET", "POST", "OPTIONS")
|
||||||
|
.allowedHeaders("*")
|
||||||
|
.allowCredentials(false)
|
||||||
|
.maxAge(3600);
|
||||||
|
```
|
||||||
|
|
||||||
|
(اترك `*` إن أردت تبسيط الاختبار الآن.)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7) Dockerfile — جديد في جذر المشروع
|
||||||
|
|
||||||
|
ملف جديد: `Dockerfile` (بجانب `pom.xml`)
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
FROM maven:3.9-eclipse-temurin-17 AS build
|
||||||
|
WORKDIR /app
|
||||||
|
COPY pom.xml .
|
||||||
|
RUN mvn -q dependency:go-offline
|
||||||
|
COPY src ./src
|
||||||
|
RUN mvn -q clean package -DskipTests
|
||||||
|
|
||||||
|
FROM eclipse-temurin:17-jre
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=build /app/target/*.jar app.jar
|
||||||
|
EXPOSE 9192
|
||||||
|
ENTRYPOINT ["java", "-jar", "app.jar"]
|
||||||
|
```
|
||||||
|
|
||||||
|
ملف جديد: `.dockerignore`
|
||||||
|
|
||||||
|
```
|
||||||
|
target/
|
||||||
|
.git/
|
||||||
|
.settings/
|
||||||
|
.classpath
|
||||||
|
.project
|
||||||
|
*.iml
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) متغيّرات البيئة (Azure) — ليست تعديل كود
|
||||||
|
|
||||||
|
عند الرفع على Azure App Service اضبط:
|
||||||
|
|
||||||
|
```
|
||||||
|
WEBSITES_PORT=9192
|
||||||
|
DB_PASSWORD=...
|
||||||
|
OPENAI_API_MISTRAL_KEY=...
|
||||||
|
AZURE_EMBEDDING_KEY=...
|
||||||
|
AZURE_SPEECH_KEY=...
|
||||||
|
HUMBEE_PASSWORD=...
|
||||||
|
```
|
||||||
|
|
||||||
|
وفعّل **Managed Identity** للتطبيق (بسبب `DefaultAzureCredential`)،
|
||||||
|
وامنحها صلاحية على مورد `homme-foundry-dev`،
|
||||||
|
واسمح لـ PostgreSQL بالاتصال (Allow Azure services).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ملخّص الملفات
|
||||||
|
|
||||||
|
| # | الملف | النوع |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 | `pom.xml` | تعديل (dependency) |
|
||||||
|
| 2 | `service/WordExportService.java` | تعديل (فرع PDF) |
|
||||||
|
| 3 | `dto/HumanLibraryRequest.java` | جديد |
|
||||||
|
| 4 | `dto/TextResponse.java` | جديد |
|
||||||
|
| 5 | `controller/HumanLibraryController.java` | جديد |
|
||||||
|
| 6 | `config/WebConfig.java` | تعديل (اختياري) |
|
||||||
|
| 7 | `Dockerfile` + `.dockerignore` | جديد |
|
||||||
|
| 8 | متغيّرات Azure | إعداد (لا كود) |
|
||||||
|
|
||||||
|
بعد هذه التعديلات: تبني بـ Docker → ترفع على Azure → تضع الروابط الثلاثة في صفحة إعدادات HÖMMA Human Library → يبدأ النموذج يعمل بالكامل (استخراج، تفريغ، حفظ).
|
||||||
@@ -82,6 +82,12 @@
|
|||||||
<artifactId>spring-boot-starter-data-jpa</artifactId>
|
<artifactId>spring-boot-starter-data-jpa</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.pdfbox</groupId>
|
||||||
|
<artifactId>pdfbox</artifactId>
|
||||||
|
<version>2.0.31</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.poi</groupId>
|
<groupId>org.apache.poi</groupId>
|
||||||
<artifactId>poi-scratchpad</artifactId>
|
<artifactId>poi-scratchpad</artifactId>
|
||||||
|
|||||||
Vendored
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,51 @@
|
|||||||
|
package com.homme.demo.config;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.web.filter.OncePerRequestFilter;
|
||||||
|
|
||||||
|
import com.github.benmanes.caffeine.cache.Cache;
|
||||||
|
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||||
|
|
||||||
|
import jakarta.servlet.FilterChain;
|
||||||
|
import jakarta.servlet.ServletException;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class RateLimitFilter extends OncePerRequestFilter {
|
||||||
|
|
||||||
|
private static final int MAX_REQUESTS_PER_MINUTE = 20;
|
||||||
|
|
||||||
|
private final Cache<String, AtomicInteger> counters = Caffeine.newBuilder()
|
||||||
|
.expireAfterWrite(Duration.ofMinutes(1))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
|
||||||
|
throws ServletException, IOException {
|
||||||
|
|
||||||
|
String path = request.getRequestURI();
|
||||||
|
|
||||||
|
boolean isProtected = path.startsWith("/knowledge-base/extract")
|
||||||
|
|| path.startsWith("/knowledge-base/transcription")
|
||||||
|
|| path.startsWith("/knowledge-base/contributions")
|
||||||
|
|| path.startsWith("/chat/");
|
||||||
|
|
||||||
|
if (isProtected) {
|
||||||
|
String ip = request.getRemoteAddr();
|
||||||
|
AtomicInteger count = counters.get(ip, k -> new AtomicInteger(0));
|
||||||
|
if (count.incrementAndGet() > MAX_REQUESTS_PER_MINUTE) {
|
||||||
|
response.setStatus(429);
|
||||||
|
response.setContentType("application/json");
|
||||||
|
response.getWriter().write("{\"error\": \"Zu viele Anfragen. Bitte später erneut versuchen.\"}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
chain.doFilter(request, response);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.homme.demo.config;
|
package com.homme.demo.config;
|
||||||
|
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
import org.springframework.web.servlet.config.annotation.CorsRegistry;
|
import org.springframework.web.servlet.config.annotation.CorsRegistry;
|
||||||
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
||||||
@@ -8,10 +9,13 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
|||||||
@Configuration
|
@Configuration
|
||||||
public class WebConfig implements WebMvcConfigurer {
|
public class WebConfig implements WebMvcConfigurer {
|
||||||
|
|
||||||
|
@Value("${homme.cors.allowed-origins}")
|
||||||
|
private String[] allowedOrigins;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void addCorsMappings(CorsRegistry registry) {
|
public void addCorsMappings(CorsRegistry registry) {
|
||||||
registry.addMapping("/**")
|
registry.addMapping("/**")
|
||||||
.allowedOriginPatterns("*")
|
.allowedOrigins(allowedOrigins)
|
||||||
.allowedMethods("GET", "POST", "OPTIONS")
|
.allowedMethods("GET", "POST", "OPTIONS")
|
||||||
.allowedHeaders("*")
|
.allowedHeaders("*")
|
||||||
.allowCredentials(false)
|
.allowCredentials(false)
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package com.homme.demo.controller;
|
package com.homme.demo.controller;
|
||||||
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.web.bind.annotation.CrossOrigin;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
@@ -14,7 +13,6 @@ import com.homme.demo.dto.ChatRequest;
|
|||||||
import com.homme.demo.service.RagService;
|
import com.homme.demo.service.RagService;
|
||||||
|
|
||||||
|
|
||||||
@CrossOrigin(origins = "${homme.cors.allowed-origins}")
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/chat")
|
@RequestMapping("/chat")
|
||||||
public class ChatController {
|
public class ChatController {
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
package com.homme.demo.controller;
|
package com.homme.demo.controller;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
import com.homme.demo.service.WordPressIngestService;
|
import com.homme.demo.service.WordPressIngestService;
|
||||||
|
|
||||||
@@ -25,4 +29,5 @@ public class IngestController {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,25 +2,36 @@ package com.homme.demo.controller;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.http.HttpStatus;
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.http.MediaType;
|
import org.springframework.http.MediaType;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
|
||||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RequestHeader;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import org.springframework.web.server.ResponseStatusException;
|
||||||
|
|
||||||
import com.homme.demo.dto.ApiResponse;
|
import com.homme.demo.dto.ApiResponse;
|
||||||
|
import com.homme.demo.dto.ContributionRequest;
|
||||||
import com.homme.demo.dto.FileProcessedDto;
|
import com.homme.demo.dto.FileProcessedDto;
|
||||||
import com.homme.demo.dto.UploadWordFileResponseDto;
|
import com.homme.demo.dto.UploadWordFileResponseDto;
|
||||||
|
import com.homme.demo.entity.Contribution;
|
||||||
import com.homme.demo.input.ContentInput;
|
import com.homme.demo.input.ContentInput;
|
||||||
|
import com.homme.demo.repository.ContributionRepository;
|
||||||
import com.homme.demo.service.AzureSpeechService;
|
import com.homme.demo.service.AzureSpeechService;
|
||||||
import com.homme.demo.service.KnowledgeIngestService;
|
import com.homme.demo.service.KnowledgeIngestService;
|
||||||
import com.homme.demo.service.WordExportService;
|
import com.homme.demo.service.WordExportService;
|
||||||
@@ -30,6 +41,9 @@ import com.homme.demo.service.WordExportService;
|
|||||||
@RequestMapping("/knowledge-base")
|
@RequestMapping("/knowledge-base")
|
||||||
public class KnowledgeIngestController {
|
public class KnowledgeIngestController {
|
||||||
|
|
||||||
|
private static final Set<String> ALLOWED_DOC_EXT = Set.of("docx", "doc", "pdf", "txt");
|
||||||
|
private static final Set<String> ALLOWED_AUDIO_EXT = Set.of("webm", "ogg", "mp3", "wav", "m4a", "mp4");
|
||||||
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private WordExportService wordExportService;
|
private WordExportService wordExportService;
|
||||||
@@ -40,28 +54,133 @@ public class KnowledgeIngestController {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private AzureSpeechService azureSpeechService;
|
private AzureSpeechService azureSpeechService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ContributionRepository contributionRepository;
|
||||||
|
|
||||||
|
@Value("${homme.admin.token}")
|
||||||
|
private String adminToken;
|
||||||
|
|
||||||
|
|
||||||
@GetMapping("/build")
|
@GetMapping("/build")
|
||||||
public List<ApiResponse<FileProcessedDto>> buildKnowledgeBase() throws Exception {
|
public List<ApiResponse<FileProcessedDto>> buildKnowledgeBase() throws Exception {
|
||||||
return knowledgeIngestService.buildKnowledgeBase();
|
return knowledgeIngestService.buildKnowledgeBase();
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/transcription")
|
@PostMapping("/transcription")
|
||||||
public String transcribe(@RequestParam("file") MultipartFile file) throws IOException{
|
public ResponseEntity<Map<String, String>> transcribe(@RequestParam("audio") MultipartFile audio) throws IOException {
|
||||||
|
|
||||||
|
String ext = getExtension(audio.getOriginalFilename());
|
||||||
|
if (!ALLOWED_AUDIO_EXT.contains(ext)) {
|
||||||
|
return new ResponseEntity<>(Map.of("error", "Nicht unterstütztes Audioformat."), HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
String text = azureSpeechService.transcribe(audio.getBytes(), audio.getOriginalFilename());
|
||||||
byte[] audioBytes = file.getBytes();
|
return ResponseEntity.ok(Map.of("text", text));
|
||||||
String fileName = file.getOriginalFilename();
|
|
||||||
return azureSpeechService.transcribe(audioBytes, fileName);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping("/extract")
|
||||||
|
public ResponseEntity<Map<String, String>> extract(@RequestParam("file") MultipartFile file) throws Exception {
|
||||||
|
|
||||||
|
String ext = getExtension(file.getOriginalFilename());
|
||||||
|
if (!ALLOWED_DOC_EXT.contains(ext)) {
|
||||||
|
return new ResponseEntity<>(Map.of("error", "Nur DOCX, DOC, PDF oder TXT erlaubt."), HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
String text = knowledgeIngestService.extractText(file.getBytes(), file.getOriginalFilename());
|
||||||
|
return ResponseEntity.ok(Map.of("text", text));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping(value = "/contributions", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||||
|
public ResponseEntity<Map<String, Object>> saveContribution(@RequestBody ContributionRequest request) {
|
||||||
|
|
||||||
|
if (request.getConsent() == null || !request.getConsent()) {
|
||||||
|
return new ResponseEntity<>(Map.of("error", "Einwilligung ist erforderlich."), HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
if (request.getName() == null || request.getName().isBlank()) {
|
||||||
|
return new ResponseEntity<>(Map.of("error", "Name ist erforderlich."), HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
if (request.getText() == null || request.getText().isBlank()) {
|
||||||
|
return new ResponseEntity<>(Map.of("error", "Text darf nicht leer sein."), HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
Contribution contribution = Contribution.builder()
|
||||||
|
.source(request.getSource())
|
||||||
|
.name(request.getName().trim())
|
||||||
|
.gender(request.getGender())
|
||||||
|
.city((request.getCity() == null || request.getCity().isBlank()) ? "Dortmund" : request.getCity().trim())
|
||||||
|
.text(request.getText())
|
||||||
|
.consent(true)
|
||||||
|
.status("PENDING")
|
||||||
|
.createdAt(LocalDateTime.now())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
Contribution saved = contributionRepository.save(contribution);
|
||||||
|
|
||||||
|
return new ResponseEntity<>(Map.of("id", saved.getId(), "status", saved.getStatus()), HttpStatus.CREATED);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@GetMapping("/contributions/pending")
|
||||||
|
public List<Contribution> pendingContributions(@RequestHeader("X-Admin-Token") String token) {
|
||||||
|
checkAdminToken(token);
|
||||||
|
return contributionRepository.findByStatus("PENDING");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@PostMapping("/contributions/{id}/approve")
|
||||||
|
public ResponseEntity<ApiResponse<FileProcessedDto>> approveContribution(@PathVariable Long id,
|
||||||
|
@RequestHeader("X-Admin-Token") String token) {
|
||||||
|
|
||||||
|
checkAdminToken(token);
|
||||||
|
try {
|
||||||
|
Contribution contribution = contributionRepository.findById(id)
|
||||||
|
.orElseThrow(() -> new RuntimeException("Beitrag nicht gefunden."));
|
||||||
|
|
||||||
|
byte[] docx = wordExportService.createInterviewDocx(contribution.getText());
|
||||||
|
|
||||||
|
String fileName = "%s_%s_%s.docx".formatted(
|
||||||
|
LocalDate.now(),
|
||||||
|
contribution.getName().replace(" ", "_"),
|
||||||
|
contribution.getCity());
|
||||||
|
|
||||||
|
UploadWordFileResponseDto upload = knowledgeIngestService.uploadWordFile(fileName, docx);
|
||||||
|
if (!upload.isSuccess()) {
|
||||||
|
return new ResponseEntity<>(ApiResponse.error("Upload fehlgeschlagen: " + upload.getMessage()),
|
||||||
|
HttpStatus.BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
ApiResponse<FileProcessedDto> processed = knowledgeIngestService.processOneFile(upload.getLink());
|
||||||
|
|
||||||
|
contribution.setStatus("APPROVED");
|
||||||
|
contributionRepository.save(contribution);
|
||||||
|
|
||||||
|
return new ResponseEntity<>(processed, HttpStatus.OK);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
return new ResponseEntity<>(ApiResponse.error("Fehler: " + e.getMessage()),
|
||||||
|
HttpStatus.INTERNAL_SERVER_ERROR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void checkAdminToken(String token) {
|
||||||
|
if (adminToken == null || adminToken.isBlank() || !adminToken.equals(token)) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "Ungültiger Admin-Token");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private String getExtension(String fileName) {
|
||||||
|
if (fileName == null || !fileName.contains(".")) return "";
|
||||||
|
return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@PostMapping(value = "/save-new-interview", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
@PostMapping(value = "/save-new-interview", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||||
public ResponseEntity<ApiResponse<FileProcessedDto>> saveNewInterview(@ModelAttribute ContentInput contentInput ) {
|
public ResponseEntity<ApiResponse<FileProcessedDto>> saveNewInterview(@ModelAttribute ContentInput contentInput ) {
|
||||||
|
|
||||||
|
|
||||||
System.out.println("rrrr ");
|
|
||||||
try {
|
try {
|
||||||
String rawText;
|
String rawText;
|
||||||
|
|
||||||
@@ -112,7 +231,3 @@ public class KnowledgeIngestController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.homme.demo.dto;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public class ContributionRequest {
|
||||||
|
|
||||||
|
private String source;
|
||||||
|
private String name;
|
||||||
|
private String gender;
|
||||||
|
private String city;
|
||||||
|
private String text;
|
||||||
|
private Boolean consent;
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package com.homme.demo.entity;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
import jakarta.persistence.Column;
|
||||||
|
import jakarta.persistence.Entity;
|
||||||
|
import jakarta.persistence.GeneratedValue;
|
||||||
|
import jakarta.persistence.GenerationType;
|
||||||
|
import jakarta.persistence.Id;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
@Entity
|
||||||
|
@Setter
|
||||||
|
@Getter
|
||||||
|
@AllArgsConstructor
|
||||||
|
@NoArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class Contribution {
|
||||||
|
|
||||||
|
@Id
|
||||||
|
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||||
|
private Long id;
|
||||||
|
|
||||||
|
private String source;
|
||||||
|
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
private String gender;
|
||||||
|
|
||||||
|
private String city;
|
||||||
|
|
||||||
|
@Column(columnDefinition = "TEXT")
|
||||||
|
private String text;
|
||||||
|
|
||||||
|
private boolean consent;
|
||||||
|
|
||||||
|
private String status; // PENDING, APPROVED, REJECTED
|
||||||
|
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package com.homme.demo.repository;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
|
import com.homme.demo.entity.Contribution;
|
||||||
|
|
||||||
|
public interface ContributionRepository extends JpaRepository<Contribution, Long> {
|
||||||
|
|
||||||
|
List<Contribution> findByStatus(String status);
|
||||||
|
}
|
||||||
@@ -13,7 +13,8 @@ import java.util.concurrent.ExecutionException;
|
|||||||
|
|
||||||
import org.apache.poi.xwpf.usermodel.XWPFDocument;
|
import org.apache.poi.xwpf.usermodel.XWPFDocument;
|
||||||
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
|
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
|
||||||
|
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||||
|
import org.apache.pdfbox.text.PDFTextStripper;
|
||||||
import org.apache.poi.hwpf.HWPFDocument;
|
import org.apache.poi.hwpf.HWPFDocument;
|
||||||
import org.apache.poi.hwpf.extractor.WordExtractor;
|
import org.apache.poi.hwpf.extractor.WordExtractor;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
@@ -249,8 +250,14 @@ public class KnowledgeIngestService {
|
|||||||
.bodyToMono(byte[].class)
|
.bodyToMono(byte[].class)
|
||||||
.block();
|
.block();
|
||||||
|
|
||||||
|
return extractText(fileBytes, downloadHref);
|
||||||
|
}
|
||||||
|
|
||||||
if(downloadHref.contains(".docx")) {
|
public String extractText(byte[] fileBytes, String fileName) throws Exception{
|
||||||
|
|
||||||
|
String name = (fileName == null) ? "" : fileName.toLowerCase();
|
||||||
|
|
||||||
|
if(name.contains(".docx")) {
|
||||||
try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) {
|
try(XWPFDocument document = new XWPFDocument(new ByteArrayInputStream(fileBytes))) {
|
||||||
|
|
||||||
StringBuilder text = new StringBuilder();
|
StringBuilder text = new StringBuilder();
|
||||||
@@ -265,14 +272,22 @@ public class KnowledgeIngestService {
|
|||||||
throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden.");
|
throw new RuntimeException("Die DDC-Datei konnte nicht gelesen werden.");
|
||||||
}
|
}
|
||||||
|
|
||||||
}else if(downloadHref.contains(".doc")) {
|
}else if(name.contains(".doc")) {
|
||||||
try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes));
|
try(HWPFDocument document = new HWPFDocument(new ByteArrayInputStream(fileBytes));
|
||||||
WordExtractor extractor = new WordExtractor(document)){
|
WordExtractor extractor = new WordExtractor(document)){
|
||||||
return extractor.getText();
|
return extractor.getText();
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if(downloadHref.contains(".txt")){
|
|
||||||
|
else if(name.endsWith(".pdf")) {
|
||||||
|
|
||||||
|
try(PDDocument doc = PDDocument.load(fileBytes)){
|
||||||
|
String rawText= new PDFTextStripper().getText(doc);
|
||||||
|
return rawText;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if(name.contains(".txt")){
|
||||||
|
|
||||||
return decodeBytes(fileBytes);
|
return decodeBytes(fileBytes);
|
||||||
|
|
||||||
@@ -293,8 +308,10 @@ public class KnowledgeIngestService {
|
|||||||
|
|
||||||
if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF)
|
if (bytes.length >= 3 && (bytes[0]&0xFF)==0xEF && (bytes[1]&0xFF)==0xBB && (bytes[2]&0xFF)==0xBF)
|
||||||
return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8);
|
return new String(bytes, 3, bytes.length-3, java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
|
||||||
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE)
|
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFF && (bytes[1]&0xFF)==0xFE)
|
||||||
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE);
|
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16LE);
|
||||||
|
|
||||||
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF)
|
if (bytes.length >= 2 && (bytes[0]&0xFF)==0xFE && (bytes[1]&0xFF)==0xFF)
|
||||||
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE);
|
return new String(bytes, 2, bytes.length-2, java.nio.charset.StandardCharsets.UTF_16BE);
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ spring.config.import=optional:classpath:/.env[.properties]
|
|||||||
server.port =9192
|
server.port =9192
|
||||||
|
|
||||||
#Origins
|
#Origins
|
||||||
homme.cors.allowed-origins=https://hmmahumanlibraryruhr1314.live-website.com
|
homme.cors.allowed-origins=http://localhost:9192,https://hmmahumanlibraryruhr1314.live-website.com,https://xn--hmma-5qa.org,https://www.xn--hmma-5qa.org
|
||||||
|
|
||||||
|
#Admin-Token fuer Freigabe von Beitraegen (Wert kommt aus .env)
|
||||||
|
homme.admin.token=${HOMME_ADMIN_TOKEN}
|
||||||
|
|
||||||
spring.datasource.url=jdbc:postgresql://homme-pg-dev-01.postgres.database.azure.com:5432/postgres?sslmode=require
|
spring.datasource.url=jdbc:postgresql://homme-pg-dev-01.postgres.database.azure.com:5432/postgres?sslmode=require
|
||||||
spring.datasource.username=mohammadadmin
|
spring.datasource.username=mohammadadmin
|
||||||
@@ -56,6 +59,6 @@ logging.level.org.springframework.security=DEBUG
|
|||||||
humbee.login.email=mohammad.zwaib@borsig11.de
|
humbee.login.email=mohammad.zwaib@borsig11.de
|
||||||
humbee.login.password=${HUMBEE_PASSWORD}
|
humbee.login.password=${HUMBEE_PASSWORD}
|
||||||
|
|
||||||
#Upload-Limits (Audio-Interviews koennen gross sein)
|
#Upload-Limits
|
||||||
spring.servlet.multipart.max-file-size=300MB
|
spring.servlet.multipart.max-file-size=25MB
|
||||||
spring.servlet.multipart.max-request-size=300MB
|
spring.servlet.multipart.max-request-size=25MB
|
||||||
@@ -64,13 +64,14 @@
|
|||||||
|
|
||||||
const audioBlob = new Blob(chunks, { type: "audio/webm" });
|
const audioBlob = new Blob(chunks, { type: "audio/webm" });
|
||||||
const formData = new FormData();
|
const formData = new FormData();
|
||||||
formData.append("file", audioBlob, "aufnahme.webm");
|
formData.append("audio", audioBlob, "aufnahme.webm");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch("/interview/transcription", { method: "POST", body: formData });
|
const response = await fetch("/knowledge-base/transcription", { method: "POST", body: formData });
|
||||||
if (!response.ok) throw new Error("Server: " + response.status);
|
if (!response.ok) throw new Error("Server: " + response.status);
|
||||||
|
|
||||||
transcript.value = await response.text();
|
const data = await response.json();
|
||||||
|
transcript.value = data.text;
|
||||||
status.textContent = "✅ Fertig – Text bitte prüfen und ggf. korrigieren.";
|
status.textContent = "✅ Fertig – Text bitte prüfen und ggf. korrigieren.";
|
||||||
saveBtn.style.display = "inline-block";
|
saveBtn.style.display = "inline-block";
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -91,14 +92,14 @@
|
|||||||
status.textContent = "⏳ Speichern läuft…";
|
status.textContent = "⏳ Speichern läuft…";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch("/interview/save", {
|
const fd = new FormData();
|
||||||
|
fd.append("text", transcript.value);
|
||||||
|
fd.append("title", document.getElementById("interviewer").value);
|
||||||
|
fd.append("city", document.getElementById("city").value);
|
||||||
|
|
||||||
|
const response = await fetch("/knowledge-base/save-new-interview", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
body: fd
|
||||||
body: JSON.stringify({
|
|
||||||
interviewr: document.getElementById("interviewer").value,
|
|
||||||
interviewTranscript: transcript.value,
|
|
||||||
city: document.getElementById("city").value
|
|
||||||
})
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const antwort = await response.json(); // ApiResponse: {success, errorMessage, data}
|
const antwort = await response.json(); // ApiResponse: {success, errorMessage, data}
|
||||||
|
|||||||
Reference in New Issue
Block a user