Merge branch 'main' of https://github.com/AraComITServicesGmbH/emotionale-robotik

.gitignore

@@ -299,3 +299,9 @@ TSWLatexianTemp*
 # option is specified. Footnotes are the stored in a file with suffix Notes.bib.
 # Uncomment the next line to have this generated file ignored.
 #*Notes.bib
+
+# Ignore python environment from hackathon
+/hackathon/env/
+
+#Ignore Python Byte code
+*pyc

hackathon/cli.py

@@ -0,0 +1,22 @@
+def print_line():
+    print("-----------------------")
+
+def print_emotions(emotions):
+    print(f"{len(emotions)} Persons with the following emotions: ")
+    print(" ".join(emotions))
+    print_line()
+
+def print_connect(address=None):
+    if address is None:
+        print(f"Connected")
+    else:
+        print(f"Connected with: {address}")
+    print_line()
+    
+def print_disconnect(address=None):
+    if address is None:
+        print(f"Disconnected")
+    else:
+        print(f"Disconnected from: {address}")
+    print_line()
+ 

hackathon/client.py

@@ -0,0 +1,44 @@
+from threading import Thread
+from queue import Queue
+import time
+
+from config import SERVER_IP, SERVER_PORT
+from communication import create_client_socket, receive_image_and_emotions
+from robot import Robot
+from own_robot import OwnRobot
+
+from cli import print_connect, print_disconnect
+
+
+def update_robot(queue: Queue, robot: Robot):
+    while True:
+        photo, emotions = queue.get()
+        with queue.mutex:
+            queue.queue.clear()
+        robot.update(photo, emotions)
+        time.sleep(robot.update_interval)
+
+
+def update_from_server(queue: Queue):
+    while True:
+        client_socket = create_client_socket()
+        try:
+            client_socket.connect((SERVER_IP, SERVER_PORT))
+            print_connect()
+            while True:
+                result = receive_image_and_emotions(client_socket)
+                if result is not None:
+                    queue.put(result)
+        except Exception: #Normally only if disconnect EOFError
+            print_disconnect()
+            client_socket.close()
+    time.sleep(1)
+
+
+if __name__ == "__main__":
+    data_queue = Queue()
+    own_robot = OwnRobot()
+    update_from_server_thread = Thread(target=update_from_server, args=(data_queue,))
+    update_robot_thread = Thread(target=update_robot, args=(data_queue, own_robot))
+    update_from_server_thread.start()
+    update_robot_thread.start()

hackathon/communication.py

@@ -0,0 +1,36 @@
+import socket
+import cv2 as cv
+import pickle
+from pickle import UnpicklingError
+
+
+def create_client_socket():
+    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 10000000)
+    return client_socket
+
+
+def create_server_socket(ip, port):
+    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    server_socket.bind((ip, port))
+    return server_socket
+
+
+def send_image_and_emotions(open_socket, address, photo, emotions):
+    ret, buffer = cv.imencode(".jpg", photo, [int(cv.IMWRITE_JPEG_QUALITY), 30])
+    x_as_bytes = pickle.dumps((buffer, emotions))
+    open_socket.sendall(x_as_bytes)
+
+    
+
+
+def receive_image_and_emotions(open_socket):
+    try:
+        x = open_socket.recvfrom(1000000)
+        data = x[0]
+        (data, emotions) = pickle.loads(data)
+        photo = cv.imdecode(data, cv.IMREAD_COLOR)
+        return (photo, emotions)
+    except UnpicklingError:
+        pass
+    return None

hackathon/config.py

@@ -0,0 +1,4 @@
+SERVER_IP = ""
+SERVER_PORT = 6666
+FRAME_RATE = 20
+UPDATE_RATE = 2

hackathon/own_robot.py

@@ -0,0 +1,17 @@
+from robot import Robot
+import cv2 as cv
+
+from cli import print_emotions
+
+import os
+
+
+class OwnRobot(Robot):
+    count = 0
+
+    def update(self, image, emotions):
+        self.count += 1
+        if self.count > 10:
+            self.count = 1
+        print_emotions(emotions)
+        cv.imwrite(os.sep.join(["img", f"ownRobot{self.count}.png"]), image)

hackathon/requirements.txt

@@ -0,0 +1,3 @@
+opencv-python-headless
+facenet-pytorch
+hsemotion

hackathon/robot.py

@@ -0,0 +1,5 @@
+class Robot:
+    update_interval = 0.5 
+    
+    def update(self,  image, emotions):
+        raise NotImplementedError()

hackathon/scripts/install.sh

@@ -0,0 +1,3 @@
+python3 -m venv env
+source env/bin/activate
+pip install -r requirements.txt

hackathon/scripts/start_client.sh

@@ -0,0 +1,2 @@
+source env/bin/activate
+python client.py

hackathon/scripts/start_server.sh

@@ -0,0 +1,2 @@
+source env/bin/activate
+python server.py

hackathon/server.py

@@ -0,0 +1,89 @@
+import cv2 as cv
+from facenet_pytorch import MTCNN
+from hsemotion.facial_emotions import HSEmotionRecognizer
+from threading import Thread
+from queue import Queue
+import time
+from config import SERVER_IP, SERVER_PORT, FRAME_RATE, UPDATE_RATE
+from communication import create_server_socket, send_image_and_emotions
+from vision import (
+    check_cuda_available,
+    check_camera_available,
+    detect_faces,
+    adjust_bounding_boxes,
+    predict_emotions_from_faces,
+)
+from cli import print_connect, print_disconnect, print_emotions
+
+
+def update_values(queue: Queue):
+    capture = cv.VideoCapture(0)
+    check_camera_available(capture)
+    device = check_cuda_available()
+    mtcnn = MTCNN(keep_all=False, post_process=False, min_face_size=40, device=device)
+    emotion_recognizer = HSEmotionRecognizer(
+        model_name="enet_b0_8_best_afew", device=device
+    )
+    previous_time = 0
+    try:
+        while True:
+            time_elapsed = time.time() - previous_time
+            if time_elapsed > 1/FRAME_RATE:
+                previous_time = time.time()
+                _, frame_bgr = capture.read()
+                frame = cv.cvtColor(frame_bgr, cv.COLOR_BGR2RGB)
+                bounding_boxes = detect_faces(mtcnn, frame)
+                bounding_boxes = adjust_bounding_boxes(
+                    bounding_boxes, frame.shape[0], frame.shape[1]
+                )
+                emotions = predict_emotions_from_faces(
+                    emotion_recognizer, frame, bounding_boxes
+                )
+                #print_emotions(emotions)
+                if queue.full():
+                    with queue.mutex:
+                        queue.queue.clear()
+                        queue.put((frame_bgr, emotions))
+                else:
+                    queue.put((frame_bgr, emotions))
+    finally:
+        capture.close()
+
+
+def update_to_clients(queue: Queue):
+    server_socket = create_server_socket(SERVER_IP, SERVER_PORT)
+    
+    try:
+        server_socket.listen(5)
+        while True:
+            client, address = server_socket.accept()
+            thread = Thread(
+                target=update_to_client, args=(client, address, queue)
+            )
+            thread.start()
+    finally:
+        server_socket.close()
+
+
+def update_to_client(client, address, queue):
+    
+    try:
+        previous_time = 0
+        print_connect(address)
+        while True:
+            time_elapsed = time.time() - previous_time
+            if time_elapsed > 1/UPDATE_RATE:
+                time_elapsed = time.time()
+                (frame_bgr, emotions) = queue.get()
+                send_image_and_emotions(client, address, frame_bgr, emotions)
+    except Exception:
+        print_disconnect(address)
+        client.close()
+
+
+if __name__ == "__main__":
+    data_queue = Queue()
+    update_values_thread = Thread(target=update_values, args=(data_queue,))
+    update_to_clients_thread = Thread(target=update_to_clients, args=(data_queue,))
+    update_values_thread.start()
+    update_to_clients_thread.start()

hackathon/vision.py

@@ -0,0 +1,56 @@
+import torch
+import numpy as np
+
+def check_cuda_available():
+    use_cuda = torch.cuda.is_available()
+    if use_cuda:
+        print("Cuda will be used for calculation")
+        return "cuda"
+    else:
+        print("Warning: CPU will be used for calculation.")
+        print("The calculation could be slowly.")
+        return "cpu"
+
+
+def check_camera_available(capture):
+    if not capture.isOpened():
+        print("Cannot open camera")
+        exit()
+
+
+def detect_faces(mtcnn, frame):
+    bounding_boxes, probs = mtcnn.detect(frame, landmarks=False)
+    if bounding_boxes is not None:
+        bounding_boxes = bounding_boxes[probs > 0.9]
+    else:
+        bounding_boxes = []
+    return bounding_boxes
+
+
+def adjust_bounding_boxes(bounding_boxes, x_limit, y_limit):
+    limited_bounding_boxes = []
+    for bounding_box in bounding_boxes:
+        box = bounding_box.astype(int)
+        x1, y1, x2, y2 = box[0:4]
+        xs = [x1, x2]
+        ys = [y1, y2]
+        xs.sort()
+        ys.sort()
+        xs[0] = np.clip(xs[0], 0, x_limit - 2)
+        xs[1] = np.clip(xs[1], 0, x_limit - 1)
+        ys[0] = np.clip(ys[0], 0, y_limit - 2)
+        ys[1] = np.clip(ys[1], 0, y_limit - 1)
+
+        limited_box = [xs[0], ys[0], xs[1], ys[1]]
+        limited_bounding_boxes.append(limited_box)
+    return limited_bounding_boxes
+
+
+def predict_emotions_from_faces(emotion_recognizer, frame, bounding_boxes):
+    emotions = []
+    for bounding_box in bounding_boxes:
+        x1, y1, x2, y2 = bounding_box[0:4]
+        face_img = frame[y1:y2, x1:x2, :]
+        emotion, _ = emotion_recognizer.predict_emotions(face_img, logits=True)
+        emotions.append(emotion)
+    return emotions

talk/arafont.sty

@@ -0,0 +1,54 @@
+\ProvidesPackage{arafont}[2023/08/08]
+\usepackage{iftex}
+
+\ifpdftex
+    % pdfLaTeX does not support directly embedding OpenType fonts
+    % We will therefore use fonts from the LaTeX font catalogue
+    % https://tug.org/FontCatalogue
+    % These fonts are pdfLaTeX ready
+    % We use OpenSans as Sans Serif and Caladea as the Serif font
+    \usepackage[defaultsans,scale=0.9]{opensans} % replaces default sans-serif font
+    \usepackage{opensans} % replaces default serif font
+    \usepackage[T1]{fontenc}
+\fi
+
+\ifxetex
+    % XeLaTeX directly supports OpenType fonts
+    % Use XeLaTeX if you would like more options with fonts
+    \usepackage{fontspec}
+    % The commands \setsansfont, \setmainfont and \setmonofont are provided by the fontspec package.
+    % These commands provide a very fine grained control over fonts.
+    % By default Beamer uses Sans-Serif mode and the font specified in \setsansfont is the font that will be applied in most places.
+    \setsansfont{Poppins}[
+        Path=../../images/fonts/,
+        Scale=0.9,
+        Extension = .ttf,
+        UprightFont=*-Regular,
+        BoldFont=*-Bold,
+        ItalicFont=*-Regular,
+        BoldItalicFont=*-Black
+    ]
+    % Other less common font styles are SlantedFont, BoldSlantedFont, SwashFont and BoldSwashFont.
+    % If your font also provides additional faces (e.g. ExtraLight, etc), fontspec also supports them.
+    % Please see the fontspec documentation Section 4.3 (Choosing additional NFSS font faces) for details.
+
+    % This is used if we use \usefonttheme{serif} in main.tex
+    % With the default \usefonttheme[onlymath]{serif}, this will control the font used for math
+    \setmainfont{Poppins}[
+        Path=../../images/fonts/,
+        Scale=0.9,
+        Extension = .ttf,
+        UprightFont=*-Regular,
+        BoldFont=*-Bold,
+        ItalicFont=*-Black,
+        BoldItalicFont=*-BoldItalicFont
+    ]
+
+    % This controls verbatim text (i.e. code)
+    \setmonofont{Poppins}[
+        Path=../../images/fonts/,
+        Scale=0.9,
+        Extension = .ttf,
+        UprightFont=*-Regular
+    ]
+\fi

talk/beamerStyle.tex

@@ -0,0 +1,304 @@
+%%%
+%%%        Project: AraCom - LaTeX Template
+%%%    Description: This is the basic LaTeX Template for all AraCom related presentations
+%%%        Version: 1.0
+%%%         Author: Robert Jeutter <robert.jeutter@aracom.de>
+%%%     Maintainer: Robert Jeutter <robert.jeutter@aracom.de>
+%%%  Creation-Date: 07.06.2023
+%%%      Copyright: (c) 2023 Robert Jeutter
+%%%      Images by AraCom IT Service
+%%%
+
+\usepackage{multirow}
+\usepackage{subfigure}
+\usepackage{etoolbox}
+\usepackage{tikz}
+\usepackage{listings}
+\usepackage{graphicx}
+\usepackage{xcolor}
+\usepackage{amsfonts, amsmath, oldgerm, lmodern, animate}
+\usepackage{verbatim}
+\usepackage{bm}
+\usepackage[T1]{fontenc}
+\graphicspath{{./images/}}
+
+\RequirePackage{arafont}
+
+\definecolor{aracomblue}{RGB}{96, 167, 192}
+\definecolor{aracomgrey}{rgb}{0.9, 0.9, 0.9}
+
+\setbeamercolor{block title}{fg=white,bg=aracomblue}
+\setbeamercolor{block body}{fg=white,bg=aracomblue}
+
+\newcommand{\themecolor}[1]{
+  \setbeamercolor{normal text}{fg=white,bg=darkgray}
+  \setbeamercolor{structure}{fg=aracomblue}
+  \setbeamercolor{block title}{fg=aracomblue,bg=aracomgrey}
+  \setbeamercolor{block body}{fg=darkgray,bg=aracomgrey}
+}
+
+\themecolor{white}
+\setbeamercolor{title}{fg=aracomblue}
+\setbeamercolor{author}{fg=white}
+\setbeamercolor{date}{fg=white}
+
+\setbeamerfont{author}{size=\scriptsize}
+\setbeamerfont{date}{size=\tiny}
+\setbeamerfont{title}{series=\bfseries, size=\fontsize{36}{40}}
+\setbeamerfont{subtitle}{series=\mdseries,size=\footnotesize}
+\setbeamerfont{frametitle}{family=\sffamily,series=\bfseries,size=\large}
+\setbeamerfont{framesubtitle}{series=\mdseries}
+\setbeamerfont{block title}{series=\centering, size=\small}
+\setbeamerfont{block body}{size=\scriptsize}
+
+% Code to get prettier boxes
+\setbeamertemplate{blocks}[rounded, shadow=true]
+
+% Bullets in several levels
+\setbeamertemplate{itemize item}{\textbullet}
+\setbeamertemplate{itemize subitem}{\textemdash}
+\setbeamertemplate{itemize subsubitem}{\ensuremath{\circ}}
+
+\newenvironment{colorblock}[3][white]{%
+  \begingroup
+  \setbeamercolor{block title}{fg=#1,bg=#2}
+  \setbeamercolor{block body} {fg=#1,bg=#2}
+  \begin{block}{#3}
+    }{%
+  \end{block}
+  \endgroup
+}
+
+% Put the logo in each slide's down right area
+\pgfdeclareimage[width=0.4\paperwidth]{araLogo}{./images/aracom_Logo-2023-white.png}
+\renewcommand{\logo}{araLogo}
+\setbeamertemplate{footline}{\hspace{0.72\paperwidth} \LARGE{www.aracom.de}}
+
+% Define frame title and subtitle layout
+\setbeamertemplate{frametitle}{
+  \begin{beamercolorbox}[leftskip=-.5cm]{frametitle}%
+    \vspace*{.1cm}\\    
+    \usebeamerfont{frametitle}\insertframetitle\\
+    \noindent\textcolor{aracomblue}{\rule{1cm}{1mm}}
+  \end{beamercolorbox}
+}
+
+% Define the title page
+\setbeamertemplate{title page}{
+  \vskip0pt plus 1filll
+  \hspace{-12mm}% Pull back the box in an inelegant way - but it works!
+  \begin{beamercolorbox}[wd=0.9\textwidth,sep=10pt,leftskip=8mm]{title}
+    {\usebeamerfont{title}\inserttitle}
+    \arabar%
+
+    {\usebeamerfont{author}\usebeamercolor[fg]{author}\insertauthor}
+
+    {\usebeamerfont{date}\usebeamercolor[fg]{date}\insertdate}
+  \end{beamercolorbox}
+  \setbeamertemplate{footline}{}
+  \vspace{-.6ex}\hspace{0.56\paperwidth}\pgfuseimage{\logo}
+}
+
+\newcommand{\arabar}[1]{%
+  \newline
+  \noindent\textcolor{aracomblue}{\rule{2cm}{2mm}}
+  \newline\noindent
+}
+
+\newcommand{\TikzSplitSlide}[1]{%
+  \rule{0.4\paperwidth}{0pt}%
+  \begin{tikzpicture}
+    \clip (-0.1\paperwidth,-0.5\paperheight) --
+    ( 0.5\paperwidth,-0.5\paperheight) --
+    ( 0.5\paperwidth, 0.5\paperheight) --
+    ( 0.1\paperwidth, 0.5\paperheight) -- cycle;
+    \node at (0.2\paperwidth,0) {%
+      \includegraphics[height=\paperheight]{#1}%
+    };
+  \end{tikzpicture}
+}
+
+\renewcommand{\maketitle}{
+  \begingroup
+  \setbeamertemplate{background}{
+    \includegraphics[height=\paperheight]{./images/background.png}
+  }
+  \vspace{-5ex}\hspace{-5ex}\begin{frame}
+    \Large\titlepage%
+  \end{frame}
+  \endgroup
+}
+
+\newenvironment{chapter}[3][]{% Args: image (optional), color, frame title
+  \begingroup
+  \themecolor{blue}
+  \ifstrequal{#2}{aracomblue}{ % Use blue text, else white
+    \setbeamercolor{frametitle}{fg=white}
+    \setbeamercolor{normal text}{fg=white,bg=#2}
+  }{
+    \setbeamercolor{frametitle}{fg=aracomblue}
+    \setbeamercolor{normal text}{fg=aracomblue,bg=#2}
+  }
+  \ifstrempty{#1}{}{\setbeamertemplate{background}{\TikzSplitSlide{#1}}}
+  \setbeamertemplate{frametitle}{%
+    \vspace*{2cm}
+    \vfill
+    \centering
+    \begin{beamercolorbox}[wd=0.55\textwidth,center,shadow=true,rounded=true]{frametitle}
+      \usebeamerfont{title}\insertframetitle\\
+      \usebeamerfont{framesubtitle}\insertframesubtitle%
+    \end{beamercolorbox}
+    \vfill
+  }
+  \begin{frame}{#3}
+    \hspace*{0.05\textwidth}%
+    \minipage{0.35\textwidth}%
+    \usebeamercolor[fg]{normal text}%
+    }{%
+    \endminipage%
+  \end{frame}
+  \endgroup
+}
+
+\newenvironment{sidepic}[2]{% Args: image, frame title
+  \begingroup
+  \setbeamertemplate{background}{%
+    \hspace*{0.6\paperwidth}%
+    \includegraphics[height=\paperheight]{#1}%
+  }
+  \setbeamertemplate{frametitle}{% Same as normal, but with right skip
+    \vspace*{-3.5ex}
+    \begin{beamercolorbox}[leftskip=2cm,rightskip=0.4\textwidth]{frametitle}%
+      \usebeamerfont{frametitle}\insertframetitle\\
+      \usebeamerfont{framesubtitle}\insertframesubtitle%
+    \end{beamercolorbox}
+  }
+  \begin{frame}{#2}
+    \minipage{0.6\textwidth}%
+    }{%
+    \endminipage%
+  \end{frame}
+  \endgroup
+}
+
+\newcommand{\strtoc}{Table of Contents}
+\newcommand{\strsubsec}{Section \thesection.\thesubsection}
+
+% TYPESETTING ELEMENTS
+
+% style of section presented in the table of contents
+\setbeamertemplate{section in toc}{$\blacktriangleright$~\inserttocsection}
+
+% style of subsection presented in the table of contents
+\setbeamertemplate{subsection in toc}{}
+\setbeamertemplate{subsection in toc}{\textcolor{white}\footnotesize\hspace{1.2 em}$\blacktriangleright$~\inserttocsubsection}
+
+% automate subtitle of each frame
+\makeatletter
+%\pretocmd\beamer@checkframetitle{\framesubtitle{\thesection\, \secname}}
+\makeatother
+
+% avoid numbering of frames that are breaked into multiply slides
+\setbeamertemplate{frametitle continuation}{}
+
+% at the begining of section, add table of contents with the current section highlighted
+\AtBeginSection[]{
+  \begingroup
+  \themecolor{blue}
+  \begin{chapter}[./images/titleimage.png]{black!90}{\insertsectionhead}
+   %\tableofcontents[currentsection]
+  \end{chapter}
+  \endgroup
+}
+
+% at the beginning of subsection, add subsection title page
+\AtBeginSubsection[]
+{
+  \begin{frame}{\,}{\thesection\, \secname}
+    \fontfamily{ptm}\selectfont
+    \centering\textsl{\textbf{\textcolor{aracomblue}{
+          \large Section \thesection.\thesubsection%
+          \vskip15pt
+          \LARGE \subsecname%
+        }}}
+  \end{frame}
+}
+
+% code block setting
+\definecolor{codegreen}{RGB}{101,218,120}
+\definecolor{codegray}{rgb}{0.5,0.5,0.5}
+\definecolor{codepurple}{rgb}{0.58,0,0.82}
+\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
+
+\lstdefinestyle{mystyle}{
+  % backgroundcolor=\color{backcolour},
+  commentstyle=\color{aracomblue},
+  keywordstyle=\color{magenta},
+  numberstyle=\tiny\color{codegray},
+  stringstyle=\color{codepurple},
+  basicstyle=\ttfamily\scriptsize,
+  breakatwhitespace=false,
+  breaklines=true,
+  captionpos=b,
+  keepspaces=true,
+  numbers=left,
+  numbersep=5pt,
+  showspaces=false,
+  showstringspaces=false,
+  showtabs=false,
+  tabsize=4,
+  xleftmargin=10pt,
+  xrightmargin=10pt,
+}
+
+\lstset{style=mystyle}
+
+% NEW COMMANDS
+
+% set colored hyperlinks command
+\newcommand{\hrefcol}[2]{\textcolor{aracomgrey}{\href{#1}{#2}}}
+\newcommand{\hlinkcol}[1]{\hrefcol{#1}{#1}}
+
+
+% centering paragraph statement
+\newcommand{\centerstate}[1]{
+  \centering
+  \begin{columns}
+    \begin{column}{0.8\textwidth}
+      #1
+    \end{column}
+  \end{columns}
+}
+
+% colored textbf
+\newcommand{\atextbf}[1]{\textbf{\textcolor{aracomblue}{#1}}}
+\newcommand{\atextsl}[1]{\textsl{\textcolor{aracomblue}{#1}}}
+\newcommand{\aemph}[1]{\emph{\textcolor{aracomblue}{#1}}}
+
+% about page
+\newcommand{\aboutpage}[2]{
+  \begingroup
+  \themecolor{blue}
+  \begin{frame}[c]{#1}{\,}
+    \centering
+    \begin{minipage}{\textwidth}
+      \usebeamercolor[fg]{normal text}
+      \centering
+      \Large \textsl{\normalsize #2}
+    \end{minipage}
+  \end{frame}
+  \endgroup
+}
+
+% bibliography page
+\newcommand{\bibliographpage}{
+  \section{References}
+
+  \begingroup
+  \themecolor{blue}
+  \begin{frame}[allowframebreaks]{References}{\,}
+    \tiny
+    \printbibliography[heading=none]
+  \end{frame}
+  \endgroup
+}