Skip to content

GitLab

Commit 51b02891 authored by Mateusz Gniewkowski's avatar Mateusz Gniewkowski
Browse files
Options

Merge branch '1-create-dockerfile' into 'master' 

Resolve "Create Dockerfile"

Closes #1

See merge request !1
parents ea0ab73b 00760d6a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 187 additions and 7304 deletions
.gitignore 0 → 100644
View file @ 51b02891
bin/
build/
.idea/
cmake-build-debug/
CMakeLists.txt 0 → 100644
View file @ 51b02891
PROJECT(spejd)
cmake_minimum_required(VERSION 3.5.0)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMakeScripts)
find_package(wsnlp 1.0.0 REQUIRED)
set(LIBS ${LIBS} ${WSNLP_LIBRARY})
find_package(spejd REQUIRED)
#include_directories(${SPEJD_INCLUDE_DIR})
set(LIBS ${LIBS} ${SPEJD_LIBRARY})
message("${SPEJD_INCLUDE_DIR}")
message("${SPEJD_LIBRARY}")
find_package(
Boost COMPONENTS
program_options random system thread
filesystem chrono atomic date_time regex)
link_directories(${Boost_LIBRARY_DIRS})
set(LIBS ${LIBS} ${Boost_LIBRARIES})
find_package(amqpcpp REQUIRED)
set(LIBS ${LIBS} amqpcpp)
include_directories("include")
add_executable(spejd src/spejd_service.cpp)
target_link_libraries(spejd ${LIBS})
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/../bin)
CMakeScripts/Findamqpcpp.cmake 0 → 100644
View file @ 51b02891
# Find the Rabbitmq C library
SET(_AMQP_REQUIRED_VARS AMQP_INCLUDE_DIR AMQP_LIBRARY )
# Find the include directories
FIND_PATH(AMQP_INCLUDE_DIR
NAMES amqpcpp.h
HINTS ${AMQP_DIR}/include /usr/include/amqpcpp
)
FIND_LIBRARY(AMQP_LIBRARY
NAMES amqpcpp
HINTS ${AMQP_DIR}/lib /usr/lib/
)
SET(AMQP_PROCESS_INCLUDES AMQP_INCLUDE_DIR)
SET(AMQP_PROCESS_LIBS AMQP_LIBRARY)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(AMQP DEFAULT_MSG ${_AMQP_REQUIRED_VARS})
\ No newline at end of file
CMakeScripts/Findspejd.cmake 0 → 100644
View file @ 51b02891
SET(_SPEJD_REQUIRED_VARS SPEJD_INCLUDE_DIR SPEJD_LIBRARY)
# Find the include directories
FIND_PATH(SPEJD_INCLUDE_DIR
NAMES spejd.h
HINTS ${SPEJD_DIR}/include /usr/include/ /usr/local/include/spejd
)
FIND_LIBRARY(SPEJD_LIBRARY
NAMES spejd
HINTS ${SPEJD_DIR}/lib /usr/lib/ /usr/local/lib
)
SET(SPEJD_PROCESS_INCLUDES SPEJD_INCLUDE_DIR)
SET(SPEJD_PROCESS_LIBS SPEJD_LIBRARY)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(SPEJD DEFAULT_MSG ${_SPEJD_REQUIRED_VARS})
CMakeScripts/Findwsnlp.cmake 0 → 100644
View file @ 51b02891
# Find the Rabbitmq C library
SET(_WSNLP_REQUIRED_VARS WSNLP_INCLUDE_DIR WSNLP_LIBRARY )
# Find the include directories
FIND_PATH(WSNLP_INCLUDE_DIR
NAMES nlpworker.h
HINTS ${WSNLP_DIR}/include /usr/include/wsnlp
)
FIND_LIBRARY(WSNLP_LIBRARY
NAMES wsnlp
HINTS ${WSNLP_DIR}/lib /usr/lib/
)
SET(WSNLP_PROCESS_INCLUDES ${WSNLP_INCLUDE_DIR})
SET(WSNLP_PROCESS_LIBS ${WSNLP_LIBRARY})
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(WSNLP DEFAULT_MSG ${_WSNLP_REQUIRED_VARS})
\ No newline at end of file
Dockerfile 0 → 100644
View file @ 51b02891
FROM clarinpl/cpp
RUN apt-get update && \
apt-get install -y \
zlib1g-dev \
software-properties-common \
pkg-config \
libxml++2.6-dev
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
WORKDIR /home/install
RUN wget http://tools.clarin-pl.eu/share/morfeusz-SGJP-linux64-20130413.tar.bz2 && \
tar -jxvf morfeusz-SGJP-linux64-20130413.tar.bz2 && \
mv libmorfeusz* /usr/local/lib/ && \
mv morfeusz /usr/local/bin/ && \
mv morfeusz.h /usr/local/include/ && \
ldconfig && \
cd /home/install && \
rm -rf morfeusz-sgjp
WORKDIR /home/install
RUN wget https://minio.clarin-pl.eu/public/share/spejd-1.4.0-fixed.tar.gz && \
tar -xzf spejd-1.4.0-fixed.tar.gz && \
cd spejd-1.4.0 && \
./configure --without-tcmalloc && \
make -j && make install && \
cd /home/install
WORKDIR /home/install/spejd
COPY src/ src/
COPY include/ include/
COPY ./CMakeLists.txt ./CMakeLists.txt
COPY CMakeScripts/ CMakeScripts/
RUN mkdir build && \
cd build && \
cmake .. && \
make -j
WORKDIR /home/worker
COPY model/ model/
RUN mv /home/install/spejd/bin/spejd . && \
cd /home && rm -rf /home/install
CMD ["./spejd"]
README.md 0 → 100644
View file @ 51b02891
# SPEJD SERVICE
### Run a container
`docker run -it -v /samba:/samba -v $(pwd)/config.ini:/home/worker/config.ini
clarinpl/spejd`
### Notes
The service depends on Spejd library (http://zil.ipipan.waw.pl/Spejd), but due
initialization bug (hardcoded config) it's slightly adjusted version is downloaded from
https://minio.clarin-pl.eu/public/share/spejd-1.4.0-fixed.tar.gz.
Also, do not use tcmalloc when building Spejd library.
config.ini
View file @ 51b02891
......@@ -2,18 +2,13 @@
; Plik zawiera konfiguracj zarwno Api usugi sieciowej jak i narzdzia.
;
; Autor: Tomasz Walkowiak
; email: tomasz.walkowiak@pwr.edu.pl
; --------- CZʌ DLA Serwisu ---------
[service]
#root = /mnt2/requests/
root = /samba/requests/
tool = spejd
rabbit_host =10.17.0.85
rabbit_user =clarin
rabbit_password =clarin123
; --------- CZʌ DLA Narzedzia ---------
[tool]
workers_number = 4
model = /home/tomekw/nlpworkers/spejd/model/config.ini
model = model/config.ini
dependancies.txt deleted 100644 → 0
View file @ ea0ab73b
AMQP-CPP
https://github.com/CopernicaMarketingSoftware/AMQP-CPP
git clone https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git
gcc >=4.8
\ No newline at end of file
include/spejd_service.hpp 0 → 100644
View file @ 51b02891
#include "nlpworker.h"
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/foreach.hpp>
#include <boost/thread.hpp>
#include <boost/filesystem.hpp>
#define BOOST_NO_CXX11_SCOPED_ENUMS
#include <string>
#include <fstream>
#include <streambuf>
#include <spejd.h>
using namespace libspejd;
using ptree = boost::property_tree::ptree;
class SpejdWorker: public NLPWorker
{
void static_init(boost::property_tree::ptree config);
void process(std::string task_path, ptree &options, std::string output_path);
void process(ptree &data, ptree &options);
static spejd *engine;
};
src/asiohandler.cpp deleted 100644 → 0
View file @ ea0ab73b
#include <iostream>
#include "asiohandler.h"
using boost::asio::ip::tcp;
class AmqpBuffer
{
public:
AmqpBuffer(size_t size) :
_data(size, 0),
_use(0)
{
}
size_t write(const char* data, size_t size)
{
if (_use == _data.size())
{
return 0;
}
const size_t length = (size + _use);
size_t write = length < _data.size() ? size : _data.size() - _use;
memcpy(_data.data() + _use, data, write);
_use += write;
return write;
}
void drain()
{
_use = 0;
}
size_t available() const
{
return _use;
}
const char* data() const
{
return _data.data();
}
void shl(size_t count)
{
assert(count < _use);
const size_t diff = _use - count;
std::memmove(_data.data(), _data.data() + count, diff);
_use = _use - count;
}
private:
std::vector<char> _data;
size_t _use;
};
AsioHandler::AsioHandler(boost::asio::io_service& ioService) :
_ioService(ioService),
_socket(ioService),
_timer(ioService),
_asioInputBuffer(ASIO_INPUT_BUFFER_SIZE, 0),
_amqpBuffer(new AmqpBuffer(ASIO_INPUT_BUFFER_SIZE * 2)),
_connection(nullptr),
_writeInProgress(false),
_connected(false),
_quit(false)
{
}
AsioHandler::~AsioHandler()
{
}
void AsioHandler::connect(const std::string& host, uint16_t port)
{
doConnect(host, port);
}
void AsioHandler::doConnect(const std::string& host, uint16_t port)
{
tcp::resolver::query query(host, std::to_string(port));
tcp::resolver::iterator iter = tcp::resolver(_ioService).resolve(query);
_timer.expires_from_now(boost::posix_time::seconds(15));
_timer.async_wait([this](const boost::system::error_code& ec){
if(!ec && !_connected)
{
std::cerr<<"Connection timed out";
_socket.cancel();
exit(1);
}
});
boost::asio::async_connect(_socket, iter,
[this](boost::system::error_code ec, tcp::resolver::iterator)
{
_connected = true;
if (!ec)
{
doRead();
if(!_outputBuffer.empty())
{
doWrite();
}
}
else
{
std::cerr<<"Connection error:"<<ec<<std::endl;
exit(1);
}
});
}
void AsioHandler::onData(
AMQP::Connection *connection, const char *data, size_t size)
{
_connection = connection;
_outputBuffer.push_back(std::vector<char>(data, data + size));
if (!_writeInProgress && _connected)
{
doWrite();
}
}
void AsioHandler::doRead()
{
_socket.async_receive(boost::asio::buffer(_asioInputBuffer),
[this](boost::system::error_code ec, std::size_t length)
{
if (!ec)
{
_amqpBuffer->write(_asioInputBuffer.data(), length);
parseData();
doRead();
}
else
{
std::cerr<<"Error reading:"<<ec<<std::endl;
exit(1);
}
});
}
void AsioHandler::doWrite()
{
_writeInProgress = true;
boost::asio::async_write(_socket,
boost::asio::buffer(_outputBuffer.front()),
[this](boost::system::error_code ec, std::size_t length )
{
if(!ec)
{
_outputBuffer.pop_front();
if(!_outputBuffer.empty())
{
doWrite();
}
else
{
_writeInProgress = false;
}
if(_quit)
{
_socket.close();
}
}
else
{
std::cerr<<"Error writing:"<<ec<<std::endl;
_socket.close();
exit(1);
}
});
}
void AsioHandler::parseData()
{
if (_connection == nullptr)
{
return;
}
const size_t count = _connection->parse(_amqpBuffer->data(),
_amqpBuffer->available());
if (count == _amqpBuffer->available())
{
_amqpBuffer->drain();
}
else if (count > 0)
{
_amqpBuffer->shl(count);
}
}
void AsioHandler::onConnected(AMQP::Connection *connection)
{
}
bool AsioHandler::connected() const
{
return _connected;
}
void AsioHandler::onError(AMQP::Connection *connection, const char *message)
{
std::cerr << "AMQP error " << message << std::endl;
}
void AsioHandler::onClosed(AMQP::Connection *connection)
{
std::cout << "AMQP closed connection" << std::endl;
_quit = true;
if (!_writeInProgress)
{
_socket.close();
}
}
src/asiohandler.h deleted 100644 → 0
View file @ ea0ab73b
#pragma once
#include <deque>
#include <vector>
#include <memory>
#include <amqpcpp.h>
#include <boost/asio.hpp>
class AmqpBuffer;
class AsioHandler: public AMQP::ConnectionHandler
{
private:
typedef std::deque<std::vector<char>> OutputBuffers;
virtual void onData(AMQP::Connection *connection, const char *data, size_t size);
virtual void onConnected(AMQP::Connection *connection);
virtual void onError(AMQP::Connection *connection, const char *message);
virtual void onClosed(AMQP::Connection *connection);
void doConnect(const std::string& host, uint16_t port);
void doRead();
void doWrite();
void parseData();
private:
boost::asio::io_service& _ioService;
boost::asio::ip::tcp::socket _socket;
boost::asio::deadline_timer _timer;
std::vector<char> _asioInputBuffer;
std::shared_ptr<AmqpBuffer> _amqpBuffer;
AMQP::Connection* _connection;
OutputBuffers _outputBuffer;
bool _writeInProgress;
bool _connected;
bool _quit;
public:
static constexpr size_t ASIO_INPUT_BUFFER_SIZE = 4*1024; //4kb
AsioHandler(boost::asio::io_service& ioService);
void connect(const std::string& host, uint16_t port);
virtual ~AsioHandler();
AsioHandler(const AsioHandler&) = delete;
AsioHandler& operator=(const AsioHandler&)=delete;
bool connected()const;
};
src/config.ini deleted 100644 → 0
View file @ ea0ab73b
; PLIK KONFIGURACYJNY WORKERA
; Plik zawiera konfiguracj zarwno Api usugi sieciowej jak i narzdzia.
;
; Autor: Tomasz Walkowiak
; email: tomasz.walkowiak@pwr.edu.pl
; --------- CZʌ DLA Serwisu ---------
[service]
#root = /mnt2/requests/
root = /samba/requests/
tool = spejd
rabbit_host =10.17.0.85
rabbit_user =clarin
rabbit_password =clarin123
; --------- CZʌ DLA Narzedzia ---------
[tool]
workers_number = 8
model = /home/tomekw/nlpworkers/spejd/model/config.ini
src/easylogging++.h deleted 100644 → 0
View file @ ea0ab73b
This diff is collapsed.
src/makefile deleted 100644 → 0
View file @ ea0ab73b
# A sample Makefile
# This Makefile demonstrates and explains
# Make Macros, Macro Expansions,
# Rules, Targets, Dependencies, Commands, Goals
# Artificial Targets, Pattern Rule, Dependency Rule.
# Comments start with a # and go to the end of the line.
CXXFLAGS=-g -std=c++11 -Wall -pedantic
# Here is a simple Make Macro.
LINK_TARGET = spejd_service
# Here is a Make Macro that uses the backslash to extend to multiple lines.
OBJS = \
asiohandler.o \
nlpworker.o \
spejd_service.o
# Here is a Make Macro defined by two Macro Expansions.
# A Macro Expansion may be treated as a textual replacement of the Make Macro.
# Macro Expansions are introduced with $ and enclosed in (parentheses).
REBUILDABLES = $(OBJS) $(LINK_TARGET)
# Here is a simple Rule (used for "cleaning" your build environment).
# It has a Target named "clean" (left of the colon ":" on the first line),
# no Dependencies (right of the colon),
# and two Commands (indented by tabs on the lines that follow).
# The space before the colon is not required but added here for clarity.
clean :
rm -f $(REBUILDABLES)
echo Clean done
# There are two standard Targets your Makefile should probably have:
# "all" and "clean", because they are often command-line Goals.
# Also, these are both typically Artificial Targets, because they don't typically
# correspond to real files named "all" or "clean".
# The rule for "all" is used to incrementally build your system.
# It does this by expressing a dependency on the results of that system,
# which in turn have their own rules and dependencies.
all : $(LINK_TARGET)
echo All done
# There is no required order to the list of rules as they appear in the Makefile.
# Make will build its own dependency tree and only execute each rule only once
# its dependencies' rules have been executed successfully.
# Here is a Rule that uses some built-in Make Macros in its command:
# $@ expands to the rule's target, in this case "test_me.exe".
# $^ expands to the rule's dependencies, in this case the three files
# main.o, test1.o, and test2.o.
$(LINK_TARGET) :$(OBJS)
g++ -std=c++11 -g -o $@ $^ -lboost_system -lboost_thread -lboost_serialization -lboost_filesystem -lamqpcpp -lspejd -pthread
# Here is a Pattern Rule, often used for compile-line.
# It says how to create a file with a .o suffix, given a file with a .cpp suffix.
# The rule's command uses some built-in Make Macros:
# $@ for the pattern-matched target
# $< for the pattern-matched dependency
%.o : %.cpp
g++ -std=c++11 -g -o $@ -c $<
# These are Dependency Rules, which are rules without any command.
# Dependency Rules indicate that if any file to the right of the colon changes,
# the target to the left of the colon should be considered out-of-date.
# The commands for making an out-of-date target up-to-date may be found elsewhere
# (in this case, by the Pattern Rule above).
# Dependency Rules are often used to capture header file dependencies.
asiohandler.o : asiohandler.h
nlpworker.o : tools.h asiohandler.h nlpworker.h
spejd_service.o : nlpworker.h
# Alternatively to manually capturing dependencies, several automated
# dependency generators exist. Here is one possibility (commented out)...
# %.dep : %.cpp
# g++ -M $(FLAGS) $< > $@
# include $(OBJS:.o=.dep)
\ No newline at end of file
src/nlpworker.cpp deleted 100644 → 0
View file @ ea0ab73b
#include <algorithm>
//#include <libwcrft/tagger.h>
#include <chrono>
#include "asiohandler.h"
#include "nlpworker.h"
//#define BOOST_NO_CXX11_SCOPED_ENUMS
//#include <boost/filesystem.hpp>
boost::property_tree::ptree init_worker()
{ boost::property_tree::ptree pt;
boost::property_tree::ini_parser::read_ini("config.ini", pt);
pt.get<std::string>("service.rabbit_host");
pt.get<std::string>("service.rabbit_user");
pt.get<std::string>("service.rabbit_password");
pt.get<std::string>("service.tool");