• R/O
  • HTTP
  • SSH
  • HTTPS

提交

标签
No Tags

Frequently used words (click to add to your profile)

javaandroidc++linuxc#objective-ccocoa誰得qtrubypythongamewindowsbathyscaphephpguic翻訳omegattwitterframeworktestbtronarduinovb.net計画中(planning stage)directxpreviewerゲームエンジンdom

docker based ci tool


Commit MetaInfo

修订版65142a1d8af111906f8c2f4fcc841cc5362977e3 (tree)
时间2019-06-06 02:19:38
作者hylom <hylom@user...>
Commiterhylom

Log Message

core-server: fix log extractor

更改概述

差异

--- a/core-server/config.py
+++ b/core-server/config.py
@@ -5,6 +5,11 @@ config = {
55 "auto_remove": True,
66 "timeout": 60,
77 },
8+ "newslash": {
9+ "container": "newslash-test",
10+ "run_mode": "start",
11+ "timeout": 300,
12+ }
813 },
914 }
1015
--- a/core-server/dockrun_pb2.py
+++ b/core-server/dockrun_pb2.py
@@ -19,13 +19,50 @@ DESCRIPTOR = _descriptor.FileDescriptor(
1919 name='dockrun.proto',
2020 package='dockrun',
2121 syntax='proto3',
22- serialized_pb=_b('\n\rdockrun.proto\x12\x07\x64ockrun\"#\n\x0eRunTaskRequest\x12\x11\n\ttask_name\x18\x01 \x01(\t\"3\n\x0cRunTaskReply\x12\x12\n\nis_succeed\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2F\n\x07\x44ockRun\x12;\n\x07RunTask\x12\x17.dockrun.RunTaskRequest\x1a\x15.dockrun.RunTaskReply\"\x00\x62\x06proto3')
22+ serialized_pb=_b('\n\rdockrun.proto\x12\x07\x64ockrun\"\xd1\x01\n\x0eRunTaskRequest\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x13\n\x0b\x63lient_name\x18\x02 \x01(\t\x12\x12\n\nuser_agent\x18\x03 \x01(\t\x12\x39\n\tparameter\x18\x04 \x03(\x0b\x32&.dockrun.RunTaskRequest.ParameterEntry\x12\x16\n\x0eparameter_type\x18\x05 \x01(\t\x1a\x30\n\x0eParameterEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x0cRunTaskReply\x12\x12\n\nis_succeed\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2F\n\x07\x44ockRun\x12;\n\x07RunTask\x12\x17.dockrun.RunTaskRequest\x1a\x15.dockrun.RunTaskReply\"\x00\x62\x06proto3')
2323 )
2424 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
2525
2626
2727
2828
29+_RUNTASKREQUEST_PARAMETERENTRY = _descriptor.Descriptor(
30+ name='ParameterEntry',
31+ full_name='dockrun.RunTaskRequest.ParameterEntry',
32+ filename=None,
33+ file=DESCRIPTOR,
34+ containing_type=None,
35+ fields=[
36+ _descriptor.FieldDescriptor(
37+ name='key', full_name='dockrun.RunTaskRequest.ParameterEntry.key', index=0,
38+ number=1, type=9, cpp_type=9, label=1,
39+ has_default_value=False, default_value=_b("").decode('utf-8'),
40+ message_type=None, enum_type=None, containing_type=None,
41+ is_extension=False, extension_scope=None,
42+ options=None),
43+ _descriptor.FieldDescriptor(
44+ name='value', full_name='dockrun.RunTaskRequest.ParameterEntry.value', index=1,
45+ number=2, type=9, cpp_type=9, label=1,
46+ has_default_value=False, default_value=_b("").decode('utf-8'),
47+ message_type=None, enum_type=None, containing_type=None,
48+ is_extension=False, extension_scope=None,
49+ options=None),
50+ ],
51+ extensions=[
52+ ],
53+ nested_types=[],
54+ enum_types=[
55+ ],
56+ options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')),
57+ is_extendable=False,
58+ syntax='proto3',
59+ extension_ranges=[],
60+ oneofs=[
61+ ],
62+ serialized_start=188,
63+ serialized_end=236,
64+)
65+
2966 _RUNTASKREQUEST = _descriptor.Descriptor(
3067 name='RunTaskRequest',
3168 full_name='dockrun.RunTaskRequest',
@@ -40,10 +77,38 @@ _RUNTASKREQUEST = _descriptor.Descriptor(
4077 message_type=None, enum_type=None, containing_type=None,
4178 is_extension=False, extension_scope=None,
4279 options=None),
80+ _descriptor.FieldDescriptor(
81+ name='client_name', full_name='dockrun.RunTaskRequest.client_name', index=1,
82+ number=2, type=9, cpp_type=9, label=1,
83+ has_default_value=False, default_value=_b("").decode('utf-8'),
84+ message_type=None, enum_type=None, containing_type=None,
85+ is_extension=False, extension_scope=None,
86+ options=None),
87+ _descriptor.FieldDescriptor(
88+ name='user_agent', full_name='dockrun.RunTaskRequest.user_agent', index=2,
89+ number=3, type=9, cpp_type=9, label=1,
90+ has_default_value=False, default_value=_b("").decode('utf-8'),
91+ message_type=None, enum_type=None, containing_type=None,
92+ is_extension=False, extension_scope=None,
93+ options=None),
94+ _descriptor.FieldDescriptor(
95+ name='parameter', full_name='dockrun.RunTaskRequest.parameter', index=3,
96+ number=4, type=11, cpp_type=10, label=3,
97+ has_default_value=False, default_value=[],
98+ message_type=None, enum_type=None, containing_type=None,
99+ is_extension=False, extension_scope=None,
100+ options=None),
101+ _descriptor.FieldDescriptor(
102+ name='parameter_type', full_name='dockrun.RunTaskRequest.parameter_type', index=4,
103+ number=5, type=9, cpp_type=9, label=1,
104+ has_default_value=False, default_value=_b("").decode('utf-8'),
105+ message_type=None, enum_type=None, containing_type=None,
106+ is_extension=False, extension_scope=None,
107+ options=None),
43108 ],
44109 extensions=[
45110 ],
46- nested_types=[],
111+ nested_types=[_RUNTASKREQUEST_PARAMETERENTRY, ],
47112 enum_types=[
48113 ],
49114 options=None,
@@ -52,8 +117,8 @@ _RUNTASKREQUEST = _descriptor.Descriptor(
52117 extension_ranges=[],
53118 oneofs=[
54119 ],
55- serialized_start=26,
56- serialized_end=61,
120+ serialized_start=27,
121+ serialized_end=236,
57122 )
58123
59124
@@ -90,19 +155,29 @@ _RUNTASKREPLY = _descriptor.Descriptor(
90155 extension_ranges=[],
91156 oneofs=[
92157 ],
93- serialized_start=63,
94- serialized_end=114,
158+ serialized_start=238,
159+ serialized_end=289,
95160 )
96161
162+_RUNTASKREQUEST_PARAMETERENTRY.containing_type = _RUNTASKREQUEST
163+_RUNTASKREQUEST.fields_by_name['parameter'].message_type = _RUNTASKREQUEST_PARAMETERENTRY
97164 DESCRIPTOR.message_types_by_name['RunTaskRequest'] = _RUNTASKREQUEST
98165 DESCRIPTOR.message_types_by_name['RunTaskReply'] = _RUNTASKREPLY
99166
100167 RunTaskRequest = _reflection.GeneratedProtocolMessageType('RunTaskRequest', (_message.Message,), dict(
168+
169+ ParameterEntry = _reflection.GeneratedProtocolMessageType('ParameterEntry', (_message.Message,), dict(
170+ DESCRIPTOR = _RUNTASKREQUEST_PARAMETERENTRY,
171+ __module__ = 'dockrun_pb2'
172+ # @@protoc_insertion_point(class_scope:dockrun.RunTaskRequest.ParameterEntry)
173+ ))
174+ ,
101175 DESCRIPTOR = _RUNTASKREQUEST,
102176 __module__ = 'dockrun_pb2'
103177 # @@protoc_insertion_point(class_scope:dockrun.RunTaskRequest)
104178 ))
105179 _sym_db.RegisterMessage(RunTaskRequest)
180+_sym_db.RegisterMessage(RunTaskRequest.ParameterEntry)
106181
107182 RunTaskReply = _reflection.GeneratedProtocolMessageType('RunTaskReply', (_message.Message,), dict(
108183 DESCRIPTOR = _RUNTASKREPLY,
@@ -112,6 +187,8 @@ RunTaskReply = _reflection.GeneratedProtocolMessageType('RunTaskReply', (_messag
112187 _sym_db.RegisterMessage(RunTaskReply)
113188
114189
190+_RUNTASKREQUEST_PARAMETERENTRY.has_options = True
191+_RUNTASKREQUEST_PARAMETERENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001'))
115192 try:
116193 # THESE ELEMENTS WILL BE DEPRECATED.
117194 # Please use the generated *_pb2_grpc.py files instead.
--- a/core-server/server.py
+++ b/core-server/server.py
@@ -24,29 +24,47 @@ OUTPUT_DIR = './results'
2424
2525 executor = concurrent.futures.ThreadPoolExecutor(max_workers=2)
2626
27-def run_task(task_name, task):
28- print("run task: '{}'".format(task_name))
27+def run_task(param, task):
28+ print("run task: '{}'".format(param["task_name"]))
2929
3030 status = {
3131 "succeeded": False,
32- "container_status": "Running"
32+ "container_status": "Running",
33+ "user_agent": param["user_agent"],
34+ "parameter": {},
35+ "parameter_type": param["parameter_type"],
3336 };
34- reporter = FileReporter(task_name, status)
35- reporter.write_status()
37+ for k in param["parameter"].keys():
38+ status["parameter"][k] = param["parameter"][k]
3639
37- image = task["image"]
38- client = docker.from_env()
39- container = client.containers.run(image=image, detach=True)
40- container.wait(timeout=task.get("timeout", 60))
40+ reporter = FileReporter(param["task_name"], status)
41+ reporter.write_status()
4142
43+ run_mode = task.get("run_mode", "run");
44+ start_epoch = time.time()
45+
46+ if run_mode == "run":
47+ image = task["image"]
48+ client = docker.from_env()
49+ container = client.containers.run(image=image, detach=True)
50+ container.wait(timeout=task.get("timeout", task.get("timeout", 60)))
51+ elif run_mode == "start":
52+ client = docker.from_env()
53+ container = client.containers.get(task["container"])
54+ container.start()
55+ container.wait(timeout=task.get("timeout", task.get("timeout", 60)))
56+
4257 print("task done.")
4358
44- lines = []
45- for line in container.logs(stream=True):
46- lines.append(line)
47- logs = b"".join(lines)
59+ #lines = []
60+ #for line in container.logs(stream=True):
61+ # lines.append(line)
62+ # logs = b"".join(lines)
63+ logs = container.logs(since=start_epoch)
64+
65+ print("extract logs done.")
4866
49- if task.get("auto_remove", True):
67+ if run_mode == "run" and task.get("auto_remove", True):
5068 container.remove()
5169 print("container removed.")
5270
@@ -56,7 +74,7 @@ def run_task(task_name, task):
5674 reporter.write_log()
5775
5876 print("done.")
59- return
77+ return True
6078
6179
6280 class FileReporter():
@@ -96,45 +114,37 @@ class FileReporter():
96114 return out_dir
97115
98116
117+def _exec_done(future):
118+ result = future.result()
119+ print("task done.")
120+ print(result)
121+
99122 class DockRun(DockRunServicer):
100123 def RunTask(self, req, context):
101124 tasks = cfg.get("tasks", {})
102125 task = tasks.get(req.task_name)
103126
127+ print("RunTask: {}".format(req.task_name))
128+
104129 if not task:
105130 message = "task '{}' not found.".format(req.task_name)
131+ print(message);
106132 return RunTaskReply(is_succeed=False, message=message)
107133
108134 # generate process
109- #self._run_task(task)
110- executor.submit(run_task, req.task_name, task)
135+ param = {
136+ "task_name": req.task_name,
137+ "client_name": req.client_name,
138+ "user_agent": req.user_agent,
139+ "parameter": req.parameter,
140+ "parameter_type": req.parameter_type,
141+ }
142+ print("execute task...")
143+ future = executor.submit(run_task, param, task)
144+ future.add_done_callback(_exec_done)
111145
112146 return RunTaskReply(is_succeed=True, message="")
113147
114- def _run_task(self, task):
115- image = task["image"]
116- client = docker.from_env()
117- container = client.containers.run(image=image, detach=True)
118- container.wait(timeout=task.get("timeout", 60))
119-
120- lines = []
121- for line in container.logs(stream=True):
122- lines.append(line)
123- logs = b"".join(lines)
124-
125- if task.get("auto_remove", True):
126- container.remove()
127-
128- report = {
129- "succeeded": False,
130- "status": None,
131- };
132-
133- reporter = FileReporter(req.task_name, report, logs)
134- reporter.write()
135-
136- return
137-
138148 def main():
139149 server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
140150 add_DockRunServicer_to_server(