Online execution of ELMO

LICENCE.txt

@@ -1,6 +1,11 @@
 MIT License
 
-Copyright (c) 2020 "Thibauld FENEUIL" <thibauld.feneuil@cryptoexperts.com>
+Copyright (c) 2020:
+    Thibauld FENEUIL
+    CryptoExperts
+    41 Boulevard des Capucines
+    75002 Paris, France
+    thibauld.feneuil@cryptoexperts.com
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -50,12 +50,12 @@ What is a _simulation project_ ? It is a project to simulate the traces of _one_
 
 To start a new project, you can use the following function.
 
-```python3
-> from elmo_online.manage import create_simulation
-> create_simulation(
->    'dilithium', # The (relative) path of the project
->    'DilithiumSimulation' # The classname of the simulation
->)
+```python
+from elmo_online.manage import create_simulation
+create_simulation(
+   'dilithium', # The (relative) path of the project
+   'DilithiumSimulation' # The classname of the simulation
+)
 ```
 
 This function will create a repository _dilithium_ with all the complete squeleton of the project. In this repository, you can find:
@@ -63,37 +63,70 @@ This function will create a repository _dilithium_ with all the complete squelet
  - The file _projectclass.py_ where there is the class of the simulation which will enable you to generate traces of the project in Python scripts;
  - A _Makefile_ ready to be used with a compiler _arm-none-eabi-gcc_.
  
-_Online ELMO_ offers a example project to you in the repository _projects/Examples_ in the module. This example is a project to generate traces of the execution of the NTT implemented in the cryptosystem [Kyber](https://pq-crystals.org/kyber/).
-
 ### List all the available simulation
 
-```python3
->from elmo_online.manage import search_simulations
->search_simulations('.')
+```python
+from elmo_online.manage import search_simulations
+search_simulations('.')
+```
+
+```python
 {'DilithiumSimulation': <class 'DilithiumSimulation'>,
  'KyberNTTSimulation': <class 'KyberNTTSimulation'>}
 ```
 
+_Online ELMO_ offers a example project to you in the repository _projects/Examples_ of the module. This example is a project to generate traces of the execution of the NTT implemented in the cryptosystem [Kyber](https://pq-crystals.org/kyber/).
+
 ### Use a simulation project
 
 Warning! Before using it, you have to compile your project thanks to the provided Makefile.
 
 ```python
-> from elmo_online.manage import get_simulation
-> KyberSimulation = get_simulation_via_classname('KyberNTTSimulation')
-> 
-> import numpy as np
-> Kyber512 = {'k': 2, 'n': 256}
-> challenges = [
->     np.ones((Kyber512['k'], Kyber512['n']), dtype=int),
-> ]
-> 
-> simulation = KyberSimulation(challenges)
-> simulation.run() # Launch the simulation
-> traces = simulation.get_traces()
-> # And now, I can draw and analyse the traces
+from elmo_online.manage import get_simulation
+KyberNTTSimulation = get_simulation_via_classname('KyberNTTSimulation')
+
+import numpy as np
+Kyber512 = {'k': 2, 'n': 256}
+challenges = [
+    np.ones((Kyber512['k'], Kyber512['n']), dtype=int),
+]
+
+simulation = KyberNTTSimulation(challenges)
+simulation.run() # Launch the simulation
+traces = simulation.get_traces()
+# And now, I can draw and analyse the traces
+```
+
+### Use a simulution project thanks to a server
+
+Sometimes, it is impossible to run the simulation thanks the simple method _run_ of the project class. Indeed, sometimes the Python script is executed in the environment where _Online ELMO_ cannot launch the ELMO tool. For example, it is the case where _Online ELMO_ is used in SageMath on Windows. On Windows, SageMath installation relies on the Cygwin POSIX emulation system and it can be a problem.
+
+To offer a solution, _Online ELMO_ can be used thanks to a link client-server. The idea is you must launch the script _run_server.py_ which will listen (by default) at port 5000 in localhost.
+
+```bash
+python3 run_server.py
+```
+
+And after, you can manipulate the projects as described in the previous section by replacing _run_ to _run_online_.
+
+```python
+from elmo_online.manage import get_simulation
+KyberNTTSimulation = get_simulation_via_classname('KyberNTTSimulation')
+
+import numpy as np
+Kyber512 = {'k': 2, 'n': 256}
+challenges = [
+    np.ones((Kyber512['k'], Kyber512['n']), dtype=int),
+]
+
+simulation = KyberNTTSimulation(challenges)
+simulation.run_online() # Launch the simulation THANKS TO A SERVER
+traces = simulation.get_traces()
+# And now, I can draw and analyse the traces
 ```
 
+Warning! Using the _run_online_ method doesn't exempt you from compiling the project with the provided Makefile.
+
 ### Use the ELMO Engine
 
 The engine exploits the model of ELMO to directly give the power consumption of an assembler instruction. In the model, to have the power consumption of an assembler instruction, it needs
@@ -110,17 +143,17 @@ The type of the instructions are:
  - "_**OTHER**_" for the other instructions.
 
 ```python
-> from elmo_online.engine import ELMOEngine, Instr
-> engine = ELMOEngine()
-> for i in range(0, 256):
->     engine.add_point(
->         (Instr.LDR, Instr.MUL, Instr.OTHER), # Types of the previous, current and next instructions
->         (0x0000, i), # Operands of the previous instructions
->         (0x2BAC, i)  # Operands of the current instructions
->     )
-> engine.run() # Compute the power consumption of all these points
-> power = engine.power # Numpy 1D array with an entry for each previous point
-> engine.reset_points() # Reset the engine to study other points
+from elmo_online.engine import ELMOEngine, Instr
+engine = ELMOEngine()
+for i in range(0, 256):
+    engine.add_point(
+        (Instr.LDR, Instr.MUL, Instr.OTHER), # Types of the previous, current and next instructions
+        (0x0000, i), # Operands of the previous instructions
+        (0x2BAC, i)  # Operands of the current instructions
+    )
+engine.run() # Compute the power consumption of all these points
+power = engine.power # Numpy 1D array with an entry for each previous point
+engine.reset_points() # Reset the engine to study other points
 ```
 
 ## Licences

executorthread.py

@@ -2,65 +2,86 @@ from servicethread import OneShotServiceThread
 import subprocess
 
 import shutil
-from project_reader import ProjectReader
 
-global_variables = {}
+import os, re
 
 class ExecutorThread(OneShotServiceThread):
     def __init__(self, ip, port, clientsocket, **kwargs):
         super().__init__(ip, port, clientsocket)
-        self.projects = kwargs['projects'] if 'projects' in kwargs else None
 
     def execute(self):
-        projects = self.projects
-        if project is None:
-            reader = ProjectReader()
-            projects = {sc.get_project_label(): sc for sc in reader.get_project_classes()}
-            print('Warning: need to research the projects.')
-        else:
-            print('Already have projects')
-        
         data = self.protocol.get_data()
         self.protocol.please_assert(data)
-        self.protocol.please_assert('project' in data)
-        self.protocol.please_assert(data['project'] in projects)
+        
+        # Set the input of ELMO
+        self.protocol.please_assert('input' in data)
+        with open('elmo/input.txt', 'w') as _input_file:
+            _input_file.write(data['input'])
+        
         self.protocol.send_ack()
+        
+        # Get the binary
+        binary_content = self.protocol.get_file()
+        with open('elmo/project.bin', 'wb') as _binary_file:
+            _binary_file.write(binary_content)
+                
+        ### Generate the traces by launching ELMO
+        command = './elmo ./project.bin'
+        cwd = './elmo/'
+        process = subprocess.Popen(
+            command, shell=True, cwd=cwd,
+            executable='/bin/bash',
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        output, error = process.communicate()
+        output = output.decode('latin-1') if output else None
+        error = error.decode('latin-1') if error else None
+        
+        if error:
+            self.protocol.send_data({
+                'output': output,
+                'error': error,
+            })
+            self.protocol.close()
+            return
+        
+        ### Get traces
+        nb_traces = output.count('TRACE NO')
 
-        # Get the project
-        project = projects[data['project']]
+        trace_filenames = []
+        for filename in os.listdir('elmo/output/traces/'):
+            if len(trace_filenames) < nb_traces:
+                if re.search(r'^trace\d+\.trc$', filename):
+                    trace_filenames.append('elmo/output/traces/{}'.format(filename))
+            else:
+                break
         
-        # Make the compilation
-        if False:
-            print('Compiling binary...')
-            
-            # Adapt the project source
-            with open('binaries/Frodo/frodo-base.c', 'r') as _src_file:
-                content = _src_file.read()
-                with open('binaries/Frodo/frodo.c', 'w') as _dst_file:
-                    _dst_file.write(content.replace('%NEED_TO_FILL%', str(n)))
-            
-            # Compile the project
-            make_directory = 'projects/{}/{}'.format(project.get_project_directory(), project.get_make_directory())
-            process = subprocess.Popen('make', shell=True, cwd=make_directory, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            output, error = process.communicate()
-            if error and ('error' in error.decode('latin-1')):
-                print("Error to compile")
-                print(error)
-                raise Exception()
-            
-            # Save last compilation data
-            global_variables[project_name] = {
-                'last_n': n,
-            }
+        assert len(trace_filenames) == nb_traces
+        results = trace_filenames
         
-        # Generate the trace by launching ELMO
-        command = './elmo ../projects/{}/{}'.format(project.get_project_directory(), project.get_binary())
-        process = subprocess.Popen(command, shell=True, cwd='elmo_online/elmo/', executable='/bin/bash', stdout=subprocess.PIPE)
-        output, error = process.communicate()
+        for i in range(len(results)):
+            with open(results[i], 'r') as _file:
+                results[i] = list(map(float,  _file.readlines()))
+                
+        ### Get asmtrace and printed data
+        asmtrace = None
+        if not ('asmtrace' in data and not data['asmtrace']):
+            with open('elmo/output/asmoutput/asmtrace00001.txt', 'r') as _file:
+                asmtrace = ''.join(_file.readlines())
+
+        printed_data = None
+        if not ('printdata' in data and not data['printdata']):
+            with open('elmo/output/printdata.txt', 'r') as _file:
+                printed_data = list(map(lambda x: int(x, 16), _file.readlines()))
 
-        # Send results
+        ### Send results
         self.protocol.send_data({
-            'output': output.decode('latin-1') if output else None,
-            'error': error.decode('latin-1') if error else None,
+            'output': output,
+            'error': error,
+            'nb_traces': nb_traces,
+            'results': results,
+            'asmtrace': asmtrace,
+            'printed_data': printed_data,
         })
         self.protocol.close()

manage.py

@@ -43,9 +43,9 @@ def search_simulations_in_module(criteria=lambda x:True):
     return search_simulations_in_repository(projects_path, criteria)
 
 def search_simulations(repository, criteria=lambda x:True):
-    projects = search_simulations_in_repository(repositories, criteria)
+    projects = search_simulations_in_repository(repository, criteria)
 
-    module_projects = search_simulations_in_module
+    module_projects = search_simulations_in_module(criteria)
     for key, project in module_projects.items():
         if key not in projects:
             projects[key] = project
@@ -142,17 +142,22 @@ def execute_simulation(project, data=None):
         }
     
     # Generate the trace by launching ELMO
-    command = './elmo {}/{}'.format(
+    command = './elmo "{}/{}"'.format(
         project.get_project_directory(),
         project.get_binary()
     )
     cwd = os.path.dirname(os.path.abspath(__file__))+'/elmo'
-    process = subprocess.Popen(command, shell=True, cwd=cwd, executable='/bin/bash', stdout=subprocess.PIPE)
+    process = subprocess.Popen(command, shell=True, cwd=cwd, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, error = process.communicate()
+    output = output.decode('latin-1') if output else None
+    error = error.decode('latin-1') if error else None
+    
+    nb_traces = output.count('TRACE NO')
 
     # Return results
-    return (
-        output.decode('latin-1') if output else None,
-        error.decode('latin-1') if error else None,
-    )
+    return {
+        'output': output,
+        'error': error,
+        'nb_traces': nb_traces,
+    }
     

project_base.py

@@ -28,22 +28,6 @@ def write_list(_input, uint16_list):
     for uint16 in uint16_list:
         write(_input, uint16)
 
-def launch_simulation(quiet=False, **kwargs):
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        s.connect(('localhost', 5000))
-        SocketTool.send_data(s, kwargs)
-        if not SocketTool.get_ack(s):
-            raise RuntimeError("NACK received: The request has been refused !")
-        else:
-            data = SocketTool.get_data(s)
-            if data['error'] and not quiet:
-                raise Exception("The simulation return an error.")
-            return data['output'], data['error']
-        s.close()
-    except IOError as err:
-        raise RuntimeError("The connection refused. Has the ELMO server been switch on ?") from err
-
 
 class SimulationProject:
     _nb_bits_for_nb_challenges = 16
@@ -89,7 +73,15 @@ class SimulationProject:
     def __init__(self, challenges=None):
         self.elmo_folder = os.path.dirname(os.path.abspath(__file__))+'/elmo'
         self.challenges = challenges
+        self.reset()
+        
+    def reset(self):
         self.is_executed = False
+        self.has_been_online = False
+
+        self._complete_asmtrace = None
+        self._complete_results = None
+        self._complete_printed_data = None
     
     def set_challenges(self, challenges):
         self.challenges = challenges
@@ -118,30 +110,66 @@ class SimulationProject:
             self.set_input_for_each_challenge(input, challenge)
             
     def run(self):
+        self.reset()
         with open('{}/input.txt'.format(self.elmo_folder), 'w') as _input:
             self.set_input(_input)
         from .manage import execute_simulation
-        execute_simulation(self)
+        res = execute_simulation(self)
         self.is_executed = True
+        self.has_been_online = False
+        return res
         
-    def run_online(self):
-        with open('{}/input.txt'.format(self.elmo_folder), 'w') as _input:
-            self.set_input(_input)
-        launch_simulation(project=self.get_project_label(), quiet=False)
+    def run_online(self, host='localhost', port=5000):
+        class TempInput:
+            def __init__(self):
+                self._buffer = ''
+            def write(self, data):
+                self._buffer += data
+            def get_string(self):
+                return self._buffer
+        
+        self.reset()
+        input = TempInput()
+        self.set_input(input)
+        
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.connect((host, port))
+            SocketTool.send_data(s, {
+                'input': input.get_string(),
+            })
+            if not SocketTool.get_ack(s):
+                raise RuntimeError("NACK received: The request has been refused !")
+            else:
+                SocketTool.send_file(s, '{}/{}'.format(self.get_project_directory(), self.get_binary()))
+                data = SocketTool.get_data(s)
+                if data['error']:
+                    raise Exception("The simulation returned an error: {}".format(data['error']))
+            s.close()
+        except IOError as err:
+            raise RuntimeError("The connection refused. Has the ELMO server been switch on ?") from err
+            
         self.is_executed = True
+        self.has_been_online = True
+        self._complete_asmtrace = data['asmtrace']
+        self._complete_results = data['results']
+        self._complete_printed_data = data['printed_data']
         
+    ### Manipulate the ASM trace
     def get_asmtrace_filename(self):
         return '{}/output/asmoutput/asmtrace00001.txt'.format(self.elmo_folder)
     
     def get_asmtrace(self):
-        with open(self.get_asmtrace_filename(), 'r') as _file:
-            return [line.strip() for line in _file.readlines()]
+        if self._complete_asmtrace is None:
+            with open(self.get_asmtrace_filename(), 'r') as _file:
+                self._complete_asmtrace = ''.join(_file.readlines())
+        
+        return self._complete_asmtrace.split('\n')        
     
     def get_indexes_of(self, condition):
-        with open(self.get_asmtrace_filename(), 'r') as _file:
-            asmtrace = _file.readlines()
-            return [i for i, instr in enumerate(asmtrace) if condition(instr)]
-    
+        return [i for i, instr in enumerate(self.get_asmtrace()) if condition(instr)]
+
+    ### Manipulate the results
     def get_number_of_traces(self):
         return len(self.challenges)
     
@@ -149,31 +177,37 @@ class SimulationProject:
         assert self.is_executed
         nb_traces = self.get_number_of_traces()
 
-        trace_filenames = []
-        for filename in os.listdir('{}/output/traces/'.format(self.elmo_folder)):
-            if re.search(r'^trace\d+\.trc$', filename):
-                trace_filenames.append('{}/output/traces/{}'.format(self.elmo_folder, filename))
-                if len(trace_filenames) >= nb_traces:
-                    break
-        
-        assert len(trace_filenames) == nb_traces
-        results = trace_filenames
-        
-        if not only_filenames:
-            for i in range(len(results)):
-                with open(results[i], 'r') as _file:
-                    if indexes is not None:
-                        results[i] = list(map(float,  _file.readlines()[indexes]))
-                    else:
-                        results[i] = list(map(float,  _file.readlines()))
+        if only_filenames and self.has_been_online:
+            raise Exception('Impossible to get the filenames for an online execution')
+
+        if only_filenames or self._complete_results is None:
+            trace_filenames = []
+            for filename in os.listdir('{}/output/traces/'.format(self.elmo_folder)):
+                if re.search(r'^trace\d+\.trc$', filename):
+                    trace_filenames.append('{}/output/traces/{}'.format(self.elmo_folder, filename))
+                    if len(trace_filenames) >= nb_traces:
+                        break
+            
+            assert len(trace_filenames) == nb_traces
+            if only_filenames:
+                return reorganise(trace_filenames) if reorganise is not None else trace_filenames
+
+            self._complete_results = []
+            for filename in trace_filenames:
+                with open(filename, 'r') as _file:
+                    self._complete_results.append(list(map(float,  _file.readlines())))
+
+        results = self._complete_results
+        if indexes is not None:
+            for i in range(len(self._complete_results)):
+                results[i] = results[i][indexes]
 
         if reorganise is not None:
             results = reorganise(results)
-
         return results
     
     def get_traces(self, reorganise=None, indexes=None):
-        results = self.get_results(only_filenames=False, reorganise=reorganise,indexes=indexes)
+        results = self.get_results(only_filenames=False, reorganise=reorganise, indexes=indexes)
 
         nb_traces = self.get_number_of_traces()
         trace_length = len(results[0])
@@ -181,21 +215,25 @@ class SimulationProject:
         traces = np.zeros((nb_traces, trace_length))
         for i in range(nb_traces):
             traces[i,:] = results[i]
-            
+        
         if reorganise is not None:
             traces = reorganise(traces)
-            
+        
         return traces
 
+    ### Manipulate the Printed Data
     def get_printed_data(self):
-        with open('{}/output/printdata.txt'.format(self.elmo_folder), 'r') as _file:
-            data = list(map(lambda x: int(x, 16), _file.readlines()))
+        if self._complete_printed_data is None:
+            with open('{}/output/printdata.txt'.format(self.elmo_folder), 'r') as _file:
+                self._complete_printed_data = list(map(lambda x: int(x, 16), _file.readlines()))
         
+        data = self._complete_printed_data
         nb_traces = self.get_number_of_traces()
         nb_data_per_trace = len(data) // nb_traces
         
         return [data[nb_data_per_trace*i:nb_data_per_trace*(i+1)] for i in range(nb_traces)]        
         
+    ### Other
     def analyse_operands(self, num_line, num_trace=1):
         num_str = str(num_trace)
         num_str = '0'*(5-len(num_str)) + num_str

protocol.py

@@ -61,7 +61,8 @@ class SocketTool:
         data = json.dumps(data)
         data = data.encode('utf-8')
         s.send(cl.convert_to_bytes(len(data))) # has to be 4 bytes
-        s.send(data)
+        for i in range(0, len(data), 1024*64):
+            s.send(data[i:i+1024*64])
         
     @classmethod
     def get_data(cl, s):
@@ -69,11 +70,7 @@ class SocketTool:
         exception_class = json.decoder.JSONDecodeError if (sys.version_info > (3, 0)) else ValueError
 
         try:
-            size = s.recv(4)
-            if not size:
-                return None
-            size = cl.bytes_to_number(size)
-            data = s.recv(size)
+            data = cl.get_file(s)
             data = data.decode('utf-8')
             data = json.loads(data)
             return data

run_server.py

@@ -1,9 +1,9 @@
-from .servicethread import ListeningThread
-from .executorthread import ExecutorThread
+from servicethread import ListeningThread
+from executorthread import ExecutorThread
 
-def do_main_program(projects):
+def do_main_program():
     global thread, stop
-    thread = ListeningThread('localhost', 5000, ExecutorThread, projects=projects)
+    thread = ListeningThread('localhost', 5000, ExecutorThread)
     thread.start()
 
 def program_cleanup(signum, frame):
@@ -14,15 +14,9 @@ def program_cleanup(signum, frame):
 thread = None
 stop = False
 
-# Information
-from .manage import search_simulations
-projects = {sc.get_project_label(): sc for sc in search_simulations_in_module().values()}
-print('Available module projects: %s' % list(projects.keys()))
-print('')
-
 # Execute
 print("Executing...")
-do_main_program(projects)
+do_main_program()
 print("Done ! And now, listening...")
 
 import signal

servicethread.py

@@ -1,5 +1,7 @@
 import threading
 from protocol import Protocol, ClosureException
+import socket
+
 
 class ServiceThread(threading.Thread):
     def run(self):