libnetworkcachetests.py 26.6 KB
Newer Older
1
import datetime
2
import errno
3
import hashlib
4
import json
5 6
import logging
import logging.handlers
Łukasz Nowak's avatar
Łukasz Nowak committed
7
import os
8
import random
9
import shutil
10
import ssl
Łukasz Nowak's avatar
Łukasz Nowak committed
11
import subprocess
12
import tempfile
13 14
import threading
import time
15
import traceback
16
import unittest
17 18
import slapos.libnetworkcache
import slapos.signature
19
import sys
20 21 22 23 24 25 26 27 28
from io import BytesIO
try:
  from http.server import BaseHTTPRequestHandler, HTTPServer
  from http.client import HTTPConnection, NOT_FOUND
  from urllib.error import HTTPError
except ImportError:
  from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
  from httplib import HTTPConnection, NOT_FOUND
  from urllib2 import HTTPError
Łukasz Nowak's avatar
Łukasz Nowak committed
29

30 31
logging.basicConfig()

32

33
class NCHandler(BaseHTTPRequestHandler):
34 35 36
  def __init__(self, request, address, server):
    self.__server = server
    self.tree = server.tree
37
    BaseHTTPRequestHandler.__init__(
38 39
      self, request, address, server)

40 41
  def handle(self):
    try:
42
      BaseHTTPRequestHandler.handle(self)
43 44 45 46
    except Exception:
      traceback.print_exc()
      raise

47 48 49 50 51 52
  def do_KILL(self):
    raise SystemExit

  def do_GET(self):
    path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
    if not (
53
      ((path == self.tree) or path.startswith(self.tree + os.path.sep))
54 55 56
      and
      os.path.exists(path)
      ):
57
      self.send_error(404)
58 59
      return
    self.send_response(200)
60 61
    with open(path, 'rb') as f:
      out = f.read()
62 63 64 65
    self.send_header('Content-Length', len(out))
    self.end_headers()
    self.wfile.write(out)

Łukasz Nowak's avatar
Łukasz Nowak committed
66
  def do_PUT(self):
67
    assert 'shadir' in self.path
68
    assert self.headers.get('content-type') == 'application/json'
69
    path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
Łukasz Nowak's avatar
Łukasz Nowak committed
70 71
    if not os.path.exists(os.path.dirname(path)):
      os.makedirs(os.path.dirname(path))
72
    data = self.rfile.read(int(self.headers.get('content-length')))
73
    cksum = hashlib.sha512(data).hexdigest()
74
    if os.path.exists(path):
75 76
      with open(path) as f:
        json_data_list = json.load(f)
Łukasz Nowak's avatar
Łukasz Nowak committed
77
    else:
78
      json_data_list = []
79
    json_data_list.append(json.loads(data.decode()))
Łukasz Nowak's avatar
Łukasz Nowak committed
80

81 82
    with open(path, 'w') as f:
      json.dump(json_data_list, f)
Łukasz Nowak's avatar
Łukasz Nowak committed
83 84 85 86
    self.send_response(201)
    self.send_header('Content-Length', str(len(cksum)))
    self.send_header('Content-Type', 'text/html')
    self.end_headers()
87
    self.wfile.write(cksum.encode())
Łukasz Nowak's avatar
Łukasz Nowak committed
88 89

  def do_POST(self):
90
    assert 'shacache' in self.path
91
    assert self.headers.get('content-type') == 'application/octet-stream'
Łukasz Nowak's avatar
Łukasz Nowak committed
92 93 94
    path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
    if not os.path.exists(path):
      os.makedirs(path)
95
    data = self.rfile.read(int(self.headers.get('content-length')))
Łukasz Nowak's avatar
Łukasz Nowak committed
96
    cksum = hashlib.sha512(data).hexdigest()
97
    path = os.path.join(path, cksum)
98

99 100 101
    # Although real server would accept the request,
    # clients should avoid uploading same content twice.
    assert not os.path.exists(path)
102 103
    with open(path, 'wb') as f:
      f.write(data)
104 105 106 107
    self.send_response(201)
    self.send_header('Content-Length', str(len(cksum)))
    self.send_header('Content-Type', 'text/html')
    self.end_headers()
108
    self.wfile.write(cksum.encode())
109

110

111 112 113
class NCHandlerPOST200(NCHandler):
  def do_POST(self):
    self.send_response(200)
114
    self.end_headers()
115

116

117 118
class NCHandlerReturnWrong(NCHandler):
  def do_POST(self):
119
    cksum = b'incorrect'
120 121 122 123 124 125
    self.send_response(201)
    self.send_header('Content-Length', str(len(cksum)))
    self.send_header('Content-Type', 'text/html')
    self.end_headers()
    self.wfile.write(cksum)

126

127
class Server(HTTPServer):
128
  def __init__(self, tree, *args):
129
    HTTPServer.__init__(self, *args)
130 131 132
    self.tree = os.path.abspath(tree)

  __run = True
133

Julien Muchembled's avatar
Julien Muchembled committed
134 135 136 137 138 139 140
  @classmethod
  def run(cls, *args, **kw):
    thread = threading.Thread(target=cls(*args, **kw).serve_forever)
    thread.daemon = True
    thread.start()
    return thread

141 142 143 144 145 146 147
  def serve_forever(self):
    while self.__run:
      self.handle_request()

  def handle_error(self, *_):
    self.__run = False

148

Łukasz Nowak's avatar
Łukasz Nowak committed
149
class OfflineTest(unittest.TestCase):
Łukasz Nowak's avatar
Łukasz Nowak committed
150 151 152 153 154 155 156
  shacache_url = 'http://localhost:1/shacache'
  shadir_url = 'http://localhost:1/shadir'
  host = 'localhost'
  port = 1
  shacache_path = '/shacache'
  shadir_path = '/shadir'

Łukasz Nowak's avatar
Łukasz Nowak committed
157
  def test_download_offline(self):
Łukasz Nowak's avatar
Łukasz Nowak committed
158 159
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache_url,
      self.shadir_url)
Łukasz Nowak's avatar
Łukasz Nowak committed
160 161 162
    self.assertRaises(IOError, nc.download, 'sha512sum')

  def test_upload_offline(self):
Łukasz Nowak's avatar
Łukasz Nowak committed
163 164
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache_url,
      self.shadir_url)
165
    self.assertRaises(IOError, nc.upload, BytesIO())
166

167

168
class OnlineMixin:
Julien Muchembled's avatar
Julien Muchembled committed
169
  handler = NCHandler
170 171
  schema = 'http'

172 173 174
  def setUp(self):
    self.host = "127.0.0.1"
    self.port = 8080
175
    self.url = '%s://%s:%s/' % (self.schema, self.host, self.port)
Julien Muchembled's avatar
Julien Muchembled committed
176 177
    self.shacache = os.environ.get('TEST_SHA_CACHE', self.url + 'shacache')
    self.shadir = os.environ.get('TEST_SHA_DIR', self.url + 'shadir')
178 179
    if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
      self.tree = tempfile.mkdtemp()
Julien Muchembled's avatar
Julien Muchembled committed
180
      self.thread = Server.run(self.tree, (self.host, self.port), self.handler)
181 182
    self.test_string = str(random.random()).encode()
    self.test_data = BytesIO(self.test_string)
Julien Muchembled's avatar
Julien Muchembled committed
183
    self.test_shasum = hashlib.sha512(self.test_string).hexdigest()
184 185
    self.handler = logging.handlers.BufferingHandler(float('inf'))
    slapos.libnetworkcache.logger.addHandler(self.handler)
186

187
  def tearDown(self):
188
    if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
189
      conn = HTTPConnection(self.host, self.port)
190
      try:
191
        conn.request('KILL', '/')
192 193
      except Exception:
        pass
194 195
      finally:
        conn.close()
196 197 198 199

      if self.thread is not None:
        self.thread.join()
      shutil.rmtree(self.tree)
200 201 202
    slapos.libnetworkcache.logger.removeHandler(self.handler)
    del self.handler

203
  def assertRaised(self, exc_type):
204
    self.assertIs(exc_type, self.handler.buffer.pop(0).args[0][0])
205

206 207 208 209 210
  def assertLog(self, msg=None):
    try:
      self.assertTrue(self.handler.buffer.pop(0).message.startswith(msg))
    except IndexError:
      self.assertEqual(msg, None)
211

212
  def select(self, nc, key, *args):
Julien Muchembled's avatar
Julien Muchembled committed
213
    try:
214
      return nc.download(next(nc.select(key))['sha512'])
215 216 217 218
    except StopIteration:
      for msg in args:
        self.assertLog(msg)
      self.assertLog()
Julien Muchembled's avatar
Julien Muchembled committed
219

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
  key = """-----BEGIN RSA PRIVATE KEY-----
MIICXQIBAAKBgQDDrOO87nSiDcXOf+xGc4Iqcdjfwd0RTOxEkO9z8mPZVg2bTPwt
/GwtPgmIC4po3bJdsCpJH21ZJwfmUpaQWIApj3odDAbRXQHWhNiw9ZPMHTCmf8Zl
yAJBxy9KI9M/fJ5RA67CJ6UYFbpF7+ZrXdkvG+0hdRX5ub0WyTPxc6kEIwIDAQAB
AoGBAIgUj1jQGKqum1bt3dps8CQmgqWyA9TJQzK3/N8MveXik5niYypz9qNMFoLX
S818CFRhdDbgNUKgAz1pSC5gbdfCDHYQTBrIt+LGpNSpdmQwReu3XoWOPZp4VWnO
uCpAkDVt+88wbxtMbZ5/ExNFs2xTO66Aad1dG12tPWoyAf4pAkEA4tCLPFNxHGPx
tluZXyWwJfVZEwLLzJ9gPkYtWrq843JuKlai2ziroubVLGSxeovBXvsjxBX95khn
U6G9Nz5EzwJBANzal8zebFdFfiN1DAyGQ4QYsmz+NsRXDbHqFVepymUId1jAFAp8
RqNt3Y78XlWOj8z5zMd4kWAR62p6LxJcyG0CQAjCaw4qXszs4zHaucKd7v6YShdc
3UgKw6nEBg5h9deG3NBPxjxXJPHGnmb3gI8uBIrJgikZfFO/ahYlwev3QKsCQGJ0
kHekMGg3cqQb6eMrd63L1L8CFSgyJsjJsfoCl1ezDoFiH40NGfCBaeP0XZmGlFSs
h73k4eoSEwDEt3dYJYECQQCBssN92KuYCOfPkJ+OV1tKdJdAsNwI13kA//A7s7qv
wHQpWKk/PLmpICMBeIiE0xT+CmCfJVOlQrqDdujganZZ
-----END RSA PRIVATE KEY-----
"""

  certificate = """-----BEGIN CERTIFICATE-----
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
MIIC7jCCAlegAwIBAgIUatGA5dEEmCL9BGYcpwEIY1l79KgwDQYJKoZIhvcNAQEL
BQAwgYgxCzAJBgNVBAYTAlVMMREwDwYDVQQIDAhCZWUgWWFyZDEYMBYGA1UECgwP
QmVlLUtlZXBlciBMdGQuMRgwFgYDVQQLDA9Ib25leSBIYXJ2ZXN0ZXIxFTATBgNV
BAMMDE1heWEgdGhlIEJlZTEbMBkGCSqGSIb3DQEJARYMTWF5YSB0aGUgQmVlMB4X
DTE4MTIwMzE1NTc1MFoXDTI4MDkwMTE1NTc1MFowgYgxCzAJBgNVBAYTAlVMMREw
DwYDVQQIDAhCZWUgWWFyZDEYMBYGA1UECgwPQmVlLUtlZXBlciBMdGQuMRgwFgYD
VQQLDA9Ib25leSBIYXJ2ZXN0ZXIxFTATBgNVBAMMDE1heWEgdGhlIEJlZTEbMBkG
CSqGSIb3DQEJARYMTWF5YSB0aGUgQmVlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
iQKBgQDDrOO87nSiDcXOf+xGc4Iqcdjfwd0RTOxEkO9z8mPZVg2bTPwt/GwtPgmI
C4po3bJdsCpJH21ZJwfmUpaQWIApj3odDAbRXQHWhNiw9ZPMHTCmf8ZlyAJBxy9K
I9M/fJ5RA67CJ6UYFbpF7+ZrXdkvG+0hdRX5ub0WyTPxc6kEIwIDAQABo1MwUTAd
BgNVHQ4EFgQUdL5Bjf3PTjtioYRUNry8OtJFDxIwHwYDVR0jBBgwFoAUdL5Bjf3P
TjtioYRUNry8OtJFDxIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOB
gQAoI3dVtQvFGvQBCGQ1MAmqzlRW0aps//GXFxc/ww4/0hkkr3OCad8i3pOfpecC
KSQX4ScodJHlfbOQ3cx0MDBSq973/8s3eMhPzV9JEsRSf19hRc1urBbqtNFkQfLN
ygUuyW4BfQm723u7T7bF3eC19J+41g6+2iHfL5YG5iygiw==
254 255 256
-----END CERTIFICATE-----
"""

257
  alternate_certificate = """-----BEGIN CERTIFICATE-----
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
MIIDUzCCAjugAwIBAgIUB/zSQmzwFmVfPuhqsvdUp78dWN4wDQYJKoZIhvcNAQEL
BQAwOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgMEERlZmF1bHQgUHJvdmluY2UxDzAN
BgNVBAoMBk5leGVkaTAeFw0xODEyMDMxNjAwMTlaFw0yODA5MDExNjAwMTlaMDkx
CzAJBgNVBAYTAkZSMRkwFwYDVQQIDBBEZWZhdWx0IFByb3ZpbmNlMQ8wDQYDVQQK
DAZOZXhlZGkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDBs0MiF/ML
QGfXiGWJEXdSYqK01papRuIl+/aWOABSCqU7lygJPPfBHAjqpMpuf3S81Wai4Lnx
fXGnwKbCLY5oP+ARaR1FxgJLJgIjQCghRncGylDYvyGV77vYofM2HY7miLWmr3J0
xpx+hP7S7a6xFreCofjhyFzWjiWoapqt55CmUef6yfZnVfWjc4zFUEWAdQh3C2Iz
D6upZB9vwdQxBZODtThy4ATVkehFp9zqkXhjH+4BQ7p1JdcIV3nqEWK3VaCGqL1c
NwsbNat9JwbtATRDiFQhkDBd/qMWx39P2sEMOSLOG13nTugDTX+5hoFrkaVeTDxB
r/cWp1Q669opAgMBAAGjUzBRMB0GA1UdDgQWBBTjfgqJoUWrxcxoVGnJGWRPFgLH
qjAfBgNVHSMEGDAWgBTjfgqJoUWrxcxoVGnJGWRPFgLHqjAPBgNVHRMBAf8EBTAD
AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBf12WXFnW3BbNLtk7V9RT5ob4CyFh+zIPq
tUQs7R8BtnemfgNAp+eFB/ZyEhy8DF0mJwrEewTr3b/p6K31HFjOUCqYDKuIbUY7
npsrJaaIxbdoG2vAMPtTjxA4iAD2tuVuSfNtDsqqHU7s4n3HmTbyPTvhr7kP1RLI
MME4ERnBbgy6Q0GBxceic+XAPBKPjqUP1DSS/qA7tagtKjhrkuViGB7frAs3hrgi
kVeHJCQRw/7QzQMf99Sp6Ii3eoYmGK4nYDwjxq7w6jeDSykata35Qs//ZVv0eStt
ZnQT1pVLar+DmUyaX9rehBM57JSnE0zvprgsVHSL0PRHH8fImdOJ
276
-----END CERTIFICATE-----
277 278
"""

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
  ca_cert = """-----BEGIN CERTIFICATE-----
MIID3zCCAsegAwIBAgIJAK6xwAnLgupDMA0GCSqGSIb3DQEBBQUAMIGFMQswCQYD
VQQGEwJQTDENMAsGA1UECAwETGFrYTELMAkGA1UEBwwCVWwxEDAOBgNVBAoMB1Bh
c2lla2ExKDAmBgNVBAMMH0F1dG9tYXRpYyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkx
HjAcBgkqhkiG9w0BCQEWD2x1a2VAbmV4ZWRpLmNvbTAeFw0xMTA4MzAwOTEwNDda
Fw00MTA4MjIwOTEwNDdaMIGFMQswCQYDVQQGEwJQTDENMAsGA1UECAwETGFrYTEL
MAkGA1UEBwwCVWwxEDAOBgNVBAoMB1Bhc2lla2ExKDAmBgNVBAMMH0F1dG9tYXRp
YyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHjAcBgkqhkiG9w0BCQEWD2x1a2VAbmV4
ZWRpLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMc1jUTwrPAC
mLUL8hwTcOpu3UxZFsAiz/XDNdNpwfEN5RzZrNeW4rJpyqCgPpXtnuQRv2Ru3c2w
oOJbL+JvGicpjleHrpU+DKw3njic7GkBYCYL+UdI4+F7a5coj+FDDHYHcY6XiIMf
3to7eJLmHeAAQe5z1MIKV4mUZGrRB494g0x2Z8rdxZkDSi8YhcwHRkMIA1p6wWY6
1ROXYdUROQ22X1mUO03fCOyyJrfuUQd3WXBtA96c9qZ1Kr8Z6qTFrTU2syRSMK1O
acZ2mSvaRYM7OuS97YqI5WBdqjQ1DRAhIsMbp42MAgEb+CxJflr6viibfw5sWqx6
WcQfHcDO8EsCAwEAAaNQME4wHQYDVR0OBBYEFIwXY1hTrkII98JBWU1o4XcLSJJm
MB8GA1UdIwQYMBaAFIwXY1hTrkII98JBWU1o4XcLSJJmMAwGA1UdEwQFMAMBAf8w
DQYJKoZIhvcNAQEFBQADggEBAEO1lmm0DTLQD0fJHYwIG+WJX2k/ThzNEE7s246m
4Hk+2ws78sdCWPkfvQHiUvCFBnfBNBfTSBToPJKaPxHi85I8VrV52/JjoA1La6pH
yr1bU9WAHv/oHRRaZcHiMqLz8/L8UM2M4VXq39BZ695tu5PI8Zu410u/62G7avht
2QjD1Xsfo5yx4LaFnTJNqq0Qn1pEVWK1QWFqntqu0l+y1zIw2R3dyxaYAkcqKO4R
euQB4LKEfrfwEBoBRK7/YXL2hewKyb/2h/5i6QazkwebLSAKOpV6LpShRbEn6O92
Ev5yliJ0c6fLy1PT0ZDevGEMigbUQo/+Bd2vIDzlrEw7mH8=
-----END CERTIFICATE-----
"""

  auth_key = """-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDA8zHDYIYDIYsq
okplkfQWac24b67/nTEXANDLjftbBGNj1/fFFK2GuVgcvdtMnqbyD1RMSMoEJa1a
uMldazh3XENTDBxUrdmEoe0Qckh9KScPjQWvxiIjaitpKEZJWZU9lK1v3EZYOweE
Ch50iiV3hIA8pbWphM8C631YxLorBGovGEDBq8q+KHBmZa3U0bByuytxCrCZP34H
NRZQM7rf+webXjqygJ+sVq2miLr5Dr2zxo44enguGsP2VZXzc5xoflQlLRgcl2IM
0u/s7H0p2XlaCBhzgjcTInYqE8I6fKCuZQWRVUkySF1omMmg3bthhGlz66aQZudK
a7HkMj4rAgMBAAECggEAG1ebGqun8fOj6/O5hTEsnKx7mYJCEzjsRu03qVDCaMBz
cSeelc/7UxcatF/3HqFw2OZxNKov7myEZ1G+Pz29b7SkWbViomFMbK4hkO4Q9aOK
RHrgbmsuVURrSGiLpUNLkcFq3mohkckzpHNmo28cJhahsXZuCsqmJyzFw3mFRCkJ
+OGaPzx0llDRkHPVMWxeQsuadDLrT5Kv4u4E7vPWmMBJ9p5Yy/xKUAX8vrtheSjo
eiHM4RW63xedmDQ6G3ofJgAiCLPCyodTtwJJnGY3ZXpCjfX0t0eyGYNkgqOM0eYw
670avZSBABCDCM7ff/w2HzxE9uY2It2VxG7Wr4WtUQKBgQDlvLMf4kE5gAtcpan4
5TJHblm1fNIVP1VMMq1WexeDnY3FHvW5FCCndJAUBW7N+ILDZVtQ+EWEcQF2hASw
6KdxBa/K/F5dLm2UUb4YLUh5ERLN+FPu7Q1jjYQtHVSsS/54YRivjV90PHnrW4G3
xljtp74+TZ/SSp3Rxnk8u8ILmQKBgQDXAemMk0LgxnDhG5PwjY6aoV6WSy4B9rT7
EXrhTDL4QZpdVlTzdGPuSSI74xbjT9B84fD8gLh+1rGVHyDlDOLduxdOuatgs2P8
zaIEBvwX15DVVINves/5gFb5FOd007LsLwfZrTSG9kLL4MR5qzl24d74z3zlB75Q
6xuU/G8SYwKBgHONyntLDouhgBWFrkzm27daJf1HX1QYmwrMoqtRFq643Mo9nFMP
cK1Jz/6CDQ3E5eDqZlf/yNepD5dRKBrjqvUKazWqYrxz0eI8i2UVwdJDaDX5ph4T
Vhyw3b7jdeeEAecCz6vdbBnHIXvkdwa82ZYQPXyRBsZ7iY4uSmTl++BhAoGAC/EX
P6+OL13WNyqI9PtnyD7eOgrC62kAdFFsOcc5rYA3SqfY4Ay+4CU/uYPLaaStN8J0
2BFuLd1Oz7GC6jXlA9u4V68ITb6o9wmUzhR1O/3FFZQ0GKUBmCIAsqTulhaMAYI7
NWPhXv2eiCRbxUY1Ut0IvVkI3s+nSmdEiOncYXECgYEAschHLdYaVl/1tvFUEubC
acabL1CwvSPh/1+xd05w2SUBZguN4q6b5zan2z34E6njRWrXbMpZ8jHroyaCFQha
CkvUOu+kXxjhuolc3LdtVssf2Zupkdwo9u07DR31t6RmJPIi/p461wgtVUI8pCK9
/59DljkBXEter7wmdnitIJg=
-----END PRIVATE KEY-----
"""

  auth_certificate = """-----BEGIN CERTIFICATE-----
MIID2TCCAsGgAwIBAgIBBDANBgkqhkiG9w0BAQUFADCBhTELMAkGA1UEBhMCUEwx
DTALBgNVBAgMBExha2ExCzAJBgNVBAcMAlVsMRAwDgYDVQQKDAdQYXNpZWthMSgw
JgYDVQQDDB9BdXRvbWF0aWMgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR4wHAYJKoZI
hvcNAQkBFg9sdWtlQG5leGVkaS5jb20wHhcNMTEwODMwMDkzOTExWhcNMjEwODI3
MDkzOTExWjBdMQswCQYDVQQGEwJQTDENMAsGA1UECAwETGFrYTEQMA4GA1UECgwH
UGFzaWVrYTENMAsGA1UEAwwEbWF5YTEeMBwGCSqGSIb3DQEJARYPbHVrZUBuZXhl
ZGkuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwPMxw2CGAyGL
KqJKZZH0FmnNuG+u/50xFwDQy437WwRjY9f3xRSthrlYHL3bTJ6m8g9UTEjKBCWt
WrjJXWs4d1xDUwwcVK3ZhKHtEHJIfSknD40Fr8YiI2oraShGSVmVPZStb9xGWDsH
hAoedIold4SAPKW1qYTPAut9WMS6KwRqLxhAwavKvihwZmWt1NGwcrsrcQqwmT9+
BzUWUDO63/sHm146soCfrFatpoi6+Q69s8aOOHp4LhrD9lWV83OcaH5UJS0YHJdi
DNLv7Ox9Kdl5WggYc4I3EyJ2KhPCOnygrmUFkVVJMkhdaJjJoN27YYRpc+umkGbn
Smux5DI+KwIDAQABo3sweTAJBgNVHRMEAjAAMCwGCWCGSAGG+EIBDQQfFh1PcGVu
U1NMIEdlbmVyYXRlZCBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU8fr/I1bRXGM7e14s
jzNA7Tx/nVswHwYDVR0jBBgwFoAUjBdjWFOuQgj3wkFZTWjhdwtIkmYwDQYJKoZI
hvcNAQEFBQADggEBAJ6q6sx+BeIi/Ia4fbjwCrw0ZahGn24fpoD7g8/eZ9XbmBMx
y4o5UlFS5LW1M0RywV0XksztU8jyQZOB8uhWw1eEdMovGqvTWmer0T0PPo14TJhN
iQh+KE90apiM8ohKJoBZ+v6s9+99YDmeW7UOw0o1wtOdT9oyT3ZbjQ57lCEUljjk
7VevyRXKYweEogzNe0lEpKiZLqiOkVtRhIY/O5eB+RYPomLmd/wWFQJrYpdRzWnj
RWYLHZ9aoSO3icgl8uvxT7WYD8fmAxXjdRd0DQVA3Jv8v8QiV+u5rxP1gcG63Bwd
08ckPaJcFIVx2H1euT4xwVDORmvuX8N3uaZLyZQ=
-----END CERTIFICATE-----
"""

359

360 361
class OnlineTest(OnlineMixin, unittest.TestCase):
  """Online tests against real HTTP server"""
362
  def test_upload(self):
363
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
364
    nc.upload(self.test_data)
365

Łukasz Nowak's avatar
Łukasz Nowak committed
366 367
  def test_upload_shadir(self):
    """Check scenario with shadir used"""
368
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
369
    urlmd5 = str(random.random())
370
    nc.upload(self.test_data, 'mykey', urlmd5=urlmd5, file_name='my file')
Łukasz Nowak's avatar
Łukasz Nowak committed
371

Łukasz Nowak's avatar
Łukasz Nowak committed
372 373
  def test_upload_shadir_select(self):
    """Check scenario with shadir used"""
374
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
Łukasz Nowak's avatar
Łukasz Nowak committed
375
    urlmd5 = str(random.random())
Łukasz Nowak's avatar
Łukasz Nowak committed
376
    key = 'somekey' + str(random.random())
377
    nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
378
    result = self.select(nc, key)
Łukasz Nowak's avatar
Łukasz Nowak committed
379 380
    self.assertEqual(result.read(), self.test_string)

381 382
  def test_upload_shadir_select_not_exists(self):
    """Check scenario with shadir used"""
383
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
384 385 386
    urlmd5 = str(random.random())
    key = 'somekey' + str(random.random())
    nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
387
    with nc:
388 389
      next(nc.select('key_another_key' + str(random.random())))
    self.assertRaised(HTTPError)
390

391 392
  def test_upload_shadir_no_filename(self):
    """Check scenario with shadir used, but not filename passed"""
393 394 395 396
    with slapos.libnetworkcache.NetworkcacheClient(self.shacache,
                                                   self.shadir) as nc:
      nc.upload(self.test_data, 'somekey', str(random.random()))
    self.assertLog('file_name and urlmd5 are required for non-generic upload')
397

398
  def test_upload_twice_same(self):
399
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
400 401 402
    nc.upload(self.test_data)
    nc.upload(self.test_data)

403
  def test_download(self):
404 405
    # prepare some test data

406
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
407 408

    # upload them
Łukasz Nowak's avatar
Łukasz Nowak committed
409
    nc.upload(self.test_data)
410 411

    # now try to download them
Łukasz Nowak's avatar
Łukasz Nowak committed
412
    result = nc.download(self.test_shasum)
413 414

    # is it correctly downloaded
Łukasz Nowak's avatar
Łukasz Nowak committed
415
    self.assertEqual(result.read(), self.test_string)
416

417
  def test_download_not_exists(self):
418
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
419 420
    try:
      nc.download(self.test_shasum)
421
      self.fail("HTTPError not raised")
422 423
    except HTTPError as error:
      self.assertEqual(error.code, NOT_FOUND)
424

Łukasz Nowak's avatar
Łukasz Nowak committed
425 426 427 428
  def test_select_signed_content(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
429
    key_file = tempfile.NamedTemporaryFile('w+')
Łukasz Nowak's avatar
Łukasz Nowak committed
430 431 432 433 434
    key_file.write(self.key)
    key_file.flush()
    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir, key_file.name, [self.certificate])
    signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
435
    result = self.select(signed_nc, key)
Łukasz Nowak's avatar
Łukasz Nowak committed
436 437
    self.assertEqual(result.read(), self.test_string)

438 439 440 441
  def test_select_signed_content_several_certificates(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
442
    key_file = tempfile.NamedTemporaryFile('w+')
443 444 445
    key_file.write(self.key)
    key_file.flush()
    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
Julien Muchembled's avatar
Julien Muchembled committed
446 447
      self.shacache, self.shadir, key_file.name,
      (self.alternate_certificate, self.certificate))
448
    signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
449
    result = self.select(signed_nc, key)
450 451
    self.assertEqual(result.read(), self.test_string)

452 453 454 455
  def test_select_signed_content_multiple(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
456
    key_file = tempfile.NamedTemporaryFile('w+')
457 458 459 460 461 462
    key_file.write(self.key)
    key_file.flush()
    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir, key_file.name, [self.certificate])
    signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
    signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
463
    result = self.select(signed_nc, key)
464 465
    self.assertEqual(result.read(), self.test_string)

466 467 468 469 470 471
  def test_select_no_entries(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
    nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir)
472 473
    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
        key, urlmd5=urlmd5, file_name=file_name))
474
    path = os.path.join(self.tree, 'shadir', key)
475
    # now remove the entry from shacache
476 477
    with open(path, 'w') as f:
      json.dump([], f)
478
    self.assertEqual(self.select(nc, key), None)
479

480 481 482 483 484 485
  def test_select_no_json_response(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
    nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir)
486 487
    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
        key, urlmd5=urlmd5, file_name=file_name))
Julien Muchembled's avatar
Julien Muchembled committed
488 489 490
    with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
      # now remove the entry from shacache
      f.write('This is not a json.')
491
    with nc:
492
      next(nc.select(key))
493
    self.assertLog('Failed to parse json response')
494

495 496 497 498 499 500
  def test_select_json_no_in_json_response(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
    nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir)
501 502
    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
        key, urlmd5=urlmd5, file_name=file_name))
Julien Muchembled's avatar
Julien Muchembled committed
503 504 505
    with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
      # now remove the entry from shacache
      f.write(json.dumps([['This is not a json.', 'signature']]))
506
    self.select(nc, key, 'Failed to parse json-in-json response')
507 508 509 510 511 512 513

  def test_select_json_in_json_no_dict(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
    nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir)
514 515
    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
        key, urlmd5=urlmd5, file_name=file_name))
Julien Muchembled's avatar
Julien Muchembled committed
516 517 518
    with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
      # now remove the entry from shacache
      f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))
519
    self.select(nc, key, 'Bad or missing sha512 in directory response')
520

Łukasz Nowak's avatar
Łukasz Nowak committed
521 522 523 524
  def test_select_signed_content_server_hacked(self):
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
    file_name = 'my file'
525
    key_file = tempfile.NamedTemporaryFile('w+')
Łukasz Nowak's avatar
Łukasz Nowak committed
526 527 528 529 530 531 532
    key_file.write(self.key)
    key_file.flush()
    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
      self.shacache, self.shadir, key_file.name, [self.certificate])

    signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
    # Hacker entered the server...
533 534 535
    path = os.path.join(self.tree, 'shadir', key)
    with open(path) as f:
      original_json = json.load(f)
Łukasz Nowak's avatar
Łukasz Nowak committed
536 537 538 539 540 541 542 543
    hacked_entry_json = json.loads(original_json[0][0])
    # ...he modifies something...
    hacked_entry_json['file'] = 'hacked'
    hacked_json = original_json[:]
    # ...and stores...
    hacked_json[0][0] = json.dumps(hacked_entry_json)
    # ...but as he has no access to key, no way to sign data..
    # hacked_json[0][1] is still a good key
544 545
    with open(path, 'w') as f:
      json.dump(hacked_json, f)
546
    self.assertEqual(self.select(signed_nc, key), None)
Łukasz Nowak's avatar
Łukasz Nowak committed
547

Łukasz Nowak's avatar
Łukasz Nowak committed
548
  def test_DirectoryNotFound_non_trustable_entry(self):
549
    key_file = tempfile.NamedTemporaryFile('w+')
550
    key_file.write(self.key)
Łukasz Nowak's avatar
Łukasz Nowak committed
551 552 553 554 555
    key_file.flush()

    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
    key = 'somekey' + str(random.random())
    urlmd5 = str(random.random())
556
    file_name = 'my file'
Łukasz Nowak's avatar
Łukasz Nowak committed
557 558
    nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
559 560
      self.shacache, self.shadir, signature_certificate_list=[
          self.certificate])
Łukasz Nowak's avatar
Łukasz Nowak committed
561
    # when no signature is used, all works ok
562
    selected = self.select(nc, key).read()
Łukasz Nowak's avatar
Łukasz Nowak committed
563 564
    self.assertEqual(selected, self.test_string)
    # but when signature is used, networkcache will complain
565
    self.assertEqual(self.select(signed_nc, key), None)
Łukasz Nowak's avatar
Łukasz Nowak committed
566 567 568 569 570 571

    # of course if proper key will be used to sign the content uploaded
    # into shacache all will work
    upload_nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache,
      self.shadir, signature_private_key_file=key_file.name)
    upload_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
572
    selected = self.select(signed_nc, key).read()
Łukasz Nowak's avatar
Łukasz Nowak committed
573
    self.assertEqual(selected, self.test_string)
Łukasz Nowak's avatar
Łukasz Nowak committed
574

575

Łukasz Nowak's avatar
Łukasz Nowak committed
576 577
@unittest.skipUnless(os.environ.get('TEST_SHA_CACHE', '') != '',
    "Requires standalone test server")
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
class OnlineTestSSLServer(OnlineMixin, unittest.TestCase):
  schema = 'https'

  def setUp(self):
    self.keyfile = tempfile.NamedTemporaryFile()
    self.keyfile.write(self.auth_key)
    self.keyfile.flush()
    self.certfile = tempfile.NamedTemporaryFile()
    self.certfile.write(self.auth_certificate)
    self.certfile.flush()
    OnlineMixin.setUp(self)

  def test_upload_to_ssl_auth_no_auth(self):
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
    try:
      nc.upload(self.test_data)
594
    except ssl.SSLError as e:
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
      self.assertTrue('alert handshake failure' in str(e))

  def test_upload_to_ssl_auth(self):
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir,
        shacache_key_file=self.keyfile.name,
        shacache_cert_file=self.certfile.name)
    nc.upload(self.test_data)

  def test_upload_with_key_with_ssl_auth_no_dir_auth(self):
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir,
        shacache_key_file=self.keyfile.name,
        shacache_cert_file=self.certfile.name)
    try:
      nc.upload(self.test_data, key=str(random.random()),
          file_name=str(random.random()), urlmd5=str(random.random()))
610
    except ssl.SSLError as e:
611 612 613 614 615 616 617 618 619 620 621 622
      self.assertTrue('alert handshake failure' in str(e))

  def test_upload_with_key_with_ssl_auth(self):
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir,
        shacache_key_file=self.keyfile.name,
        shacache_cert_file=self.certfile.name,
        shadir_key_file=self.keyfile.name,
        shadir_cert_file=self.certfile.name)
    nc.upload(self.test_data, key=str(random.random()),
          file_name=str(random.random()), urlmd5=str(random.random()))


623
class OnlineTestPOST200(OnlineMixin, unittest.TestCase):
Julien Muchembled's avatar
Julien Muchembled committed
624
  handler = NCHandlerPOST200
625 626 627

  def test_upload_wrong_return_code(self):
    """Check reaction on HTTP return code different then 201"""
628
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
629
    self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
630 631
      self.test_data)

632 633

class OnlineTestWrongChecksum(OnlineMixin, unittest.TestCase):
Julien Muchembled's avatar
Julien Muchembled committed
634
  handler = NCHandlerReturnWrong
635 636 637

  def test_upload_wrong_return_sha(self):
    """Check reaction in case of wrong sha returned"""
638
    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
639
    self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
640 641
      self.test_data)

642

Łukasz Nowak's avatar
Łukasz Nowak committed
643
class GenerateSignatureScriptTest(unittest.TestCase):
Łukasz Nowak's avatar
Łukasz Nowak committed
644
  ''' Class which must test the signature.py script. '''
645 646
  def setUp(self):
    self.key = os.path.join(tempfile.gettempdir(), tempfile.gettempprefix() +
Łukasz Nowak's avatar
Łukasz Nowak committed
647
      str(random.random()))
648
    self.certificate = os.path.join(tempfile.gettempdir(), tempfile.gettempprefix()
Łukasz Nowak's avatar
Łukasz Nowak committed
649
      + str(random.random()))
650
    self.common_name = str(random.random())
Łukasz Nowak's avatar
Łukasz Nowak committed
651

652
  def tearDown(self):
Julien Muchembled's avatar
Julien Muchembled committed
653
    for f in self.key, self.certificate:
654 655
      if os.path.lexists(f):
        os.unlink(f)
656 657 658 659

  def test_generate_certificate(self):
    slapos.signature.generateCertificate(self.certificate, self.key,
      self.common_name)
660
    today = datetime.date.today()
661
    result = subprocess.check_output(['openssl', 'x509', '-noout', '-subject',
662
      '-in', self.certificate], universal_newlines=True)
663
    self.assertEqual('subject=CN = %s' % self.common_name, result.strip())
664
    result = subprocess.check_output(['openssl', 'x509', '-noout', '-enddate',
665 666
      '-in', self.certificate], universal_newlines=True)
    self.assertIn(' %s ' % (today.year + 100), result)
667 668

  def test_generate_key_exists(self):
Julien Muchembled's avatar
Julien Muchembled committed
669 670 671
    with tempfile.NamedTemporaryFile() as key:
      self.assertRaises(ValueError, slapos.signature.generateCertificate,
        self.certificate, key.name, self.common_name)
672 673

  def test_generate_cert_exists(self):
Julien Muchembled's avatar
Julien Muchembled committed
674 675 676
    with tempfile.NamedTemporaryFile() as cert:
      self.assertRaises(ValueError, slapos.signature.generateCertificate,
        cert.name, self.key, self.common_name)