webservice v1

searchengine/.gitignore

@@ -0,0 +1 @@
+__pycache__

searchengine/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.8-slim-buster AS uwsgi
+
+ENV PYTHONUNBUFFERED 1
+WORKDIR /app
+
+RUN apt-get -y update && apt-get install -y gcc
+
+COPY requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . /app
+
+EXPOSE 8000
+
+CMD uwsgi --ini uwsgi.ini

searchengine/README.md

@@ -0,0 +1,46 @@
+```
+$ curl -X POST -H 'Content-Type: application/json' --data '{"symbols": {"strncpy": "db0", "strcat": "0x000000000d800"}}' 'https://libc.rip/api/find'
+[
+  {
+    "buildid": "d3cf764b2f97ac3efe366ddd07ad902fb6928fd7",
+    "download_url": "https://libc.rip/download/libc6_2.27-3ubuntu1.2_amd64.so",
+    "id": "libc6_2.27-3ubuntu1.2_amd64",
+    "md5": "35ef4ffc9c6ad7ffd1fd8c16f14dc766",
+    "sha1": "a22321cd65f28f70cf321614fdfd22f36ecd0afe",
+    "sha256": "f0ad9639b2530741046e06c96270b25da2339b6c15a7ae46de8fb021b3c4f529",
+    "symbols": {
+      "__libc_start_main_ret": "0x21b97",
+      "dup2": "0x110ab0",
+      "printf": "0x64f00",
+      "puts": "0x80a30",
+      "read": "0x110180",
+      "str_bin_sh": "0x1b40fa",
+      "strcat": "0x9d800",
+      "strncpy": "0x9ddb0",
+      "system": "0x4f4e0",
+      "write": "0x110250"
+    }
+  }
+]
+
+$ curl -X POST -H 'Content-Type: application/json' --data '{"symbols": ["strcat"]}' 'https://libc.rip/api/libc/libc6_2.27-3ubuntu1.2_amd64'
+{
+  "buildid": "d3cf764b2f97ac3efe366ddd07ad902fb6928fd7",
+  "download_url": "https://libc.rip/download/libc6_2.27-3ubuntu1.2_amd64.so",
+  "id": "libc6_2.27-3ubuntu1.2_amd64",
+  "md5": "35ef4ffc9c6ad7ffd1fd8c16f14dc766",
+  "sha1": "a22321cd65f28f70cf321614fdfd22f36ecd0afe",
+  "sha256": "f0ad9639b2530741046e06c96270b25da2339b6c15a7ae46de8fb021b3c4f529",
+  "symbols": {
+    "__libc_start_main_ret": "0x21b97",
+    "dup2": "0x110ab0",
+    "printf": "0x64f00",
+    "puts": "0x80a30",
+    "read": "0x110180",
+    "str_bin_sh": "0x1b40fa",
+    "strcat": "0x9d800",
+    "system": "0x4f4e0",
+    "write": "0x110250"
+  }
+}
+```

searchengine/api.yml

@@ -0,0 +1,92 @@
+openapi: 3.0.3
+info:
+  version: 1.0.0
+  title: libc-database search engine
+  description: ''
+paths:
+  /find:
+    post:
+      operationId: app.find
+      tags:
+      - libcsearch
+      description: |-
+        Look up libc by various attributes
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/Libc'
+        required: true
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/Libc'
+          description: ''
+
+  /libc/{id}:
+    post:
+      operationId: app.dump
+      tags:
+      - libcsearch
+      parameters:
+      - in: path
+        name: id
+        schema:
+          type: string
+        required: true
+      description: |-
+        Dump libc symbols
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/DumpRequest'
+        required: true
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Libc'
+          description: ''
+
+components:
+  schemas:
+    Libc:
+      type: object
+      properties:
+        md5:
+          type: string
+        sha1:
+          type: string
+        sha256:
+          type: string
+        buildid:
+          type: string
+        id:
+          type: string
+        symbols:
+          type: object
+          additionalProperties:
+            type: string
+            pattern: '^(0x)?[a-fA-F0-9]+$'
+        download_url:
+          type: string
+          format: url
+          readOnly: true
+
+    DumpRequest:
+      type: object
+      properties:
+        symbols:
+          type: array
+          items:
+            type: string
+
+
+servers:
+  - url: https://libc.rip/api

searchengine/app.py

@@ -0,0 +1,99 @@
+import logging
+from functools import lru_cache
+
+import connexion
+from elasticsearch import Elasticsearch
+
+import config
+
+
+es = Elasticsearch(hosts=[config.ES_HOST])
+log = logging.getLogger('wsgi')
+
+log.info(f'Using elasticsearch server {config.ES_HOST}, index {config.ES_INDEX_NAME}')
+
+
+@lru_cache(maxsize=2000)
+def get_symbols(id):
+    syms = {}
+    with open(f'{config.DB_DIR}/{id}.symbols') as f:
+        for line in f:
+            if not line.strip():
+                continue
+            name, addr = line.split()
+            addr = int(addr, 16)
+            syms[name] = addr
+    return syms
+
+
+def find(body, extra_symbols=[]):
+    filters = []
+
+    for h in ('id', 'md5', 'sha1', 'sha256', 'buildid'):
+        if h in body:
+            filters.append({'match': {h: body[h]}})
+
+    symbol_filters = body.get('symbols')
+    if symbol_filters:
+        terms = []
+        for sym, addr in symbol_filters.items():
+            addr = int(addr, 16)
+            term = f'{sym}@{addr & 0xfff:03x}'
+            filters.append({'term': {'symbols': term}})
+
+
+    if not filters:
+        return connexion.problem(
+            status=400,
+            title='Bad request',
+            detail='must provide at least one filter',
+        )
+
+    query = {"bool": {"filter": filters}}
+    res = es.search(index=config.ES_INDEX_NAME, body={"query": query})
+
+    libcs = []
+    for hit in res['hits']['hits']:
+        doc = hit['_source']
+        id = doc['id']
+        syms = get_symbols(id)
+
+        result_symbols = {}
+
+        names = list(config.DEFAULT_SYMBOLS) + extra_symbols
+        if symbol_filters:
+            names += symbol_filters.keys()
+        for name in names:
+            if name in syms:
+                result_symbols[name] = f'{syms[name]:#x}'
+
+        libcs.append({
+            'id': id,
+            'buildid': doc.get('buildid'),
+            'sha1': doc.get('sha1'),
+            'md5': doc.get('md5'),
+            'sha256': doc.get('sha256'),
+            'symbols': result_symbols,
+            'download_url': config.DOWNLOAD_URL.format(id),
+        })
+    return libcs
+
+
+def dump(id, body):
+    res = find({'id': id}, extra_symbols=body['symbols'])
+    if not res:
+        return connexion.problem(
+            status=404,
+            title='Not found',
+            detail=f'Unknown ID: {id}'
+        )
+
+    return res[0]
+
+
+app = connexion.App(__name__, specification_dir='.')
+app.add_api('api.yml')
+
+
+if __name__ == '__main__':
+    app.run(port=8080, host='127.0.0.1', debug=True)

searchengine/config.py

@@ -0,0 +1,21 @@
+import os
+
+ES_INDEX_NAME = 'libcsearch'
+
+# ES_HOST = 'localhost'
+ES_HOST = 'es01'
+
+# DB_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../db'
+DB_DIR = '/db'
+
+DEFAULT_SYMBOLS = [
+    '__libc_start_main_ret',
+    'system',
+    'dup2',
+    'str_bin_sh',
+    'read',
+    'write',
+    'puts',
+    'printf',
+]
+DOWNLOAD_URL = 'https://libc.rip/download/{}.so'

searchengine/docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3.4'
+services:
+
+  uwsgi:
+    build:
+      context: .
+      target: uwsgi
+    volumes:
+      - ../db:/db
+    ports:
+      - 127.0.0.1:8000:8000
+    networks:
+      - libcsearch
+    restart: always
+    logging:
+      driver: syslog
+      options:
+        tag: uwsgi
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0
+    container_name: es01
+    environment:
+      - node.name=es01
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data01:/usr/share/elasticsearch/data
+    ports:
+      - 127.0.0.1:9200:9200
+    networks:
+      - libcsearch
+    restart: always
+    logging:
+      driver: syslog
+      options:
+        tag: elasticsearch
+
+volumes:
+  data01:
+    driver: local
+
+networks:
+  libcsearch: