Skip to content
Snippets Groups Projects
Select Git revision
  • master
1 result

gpt2_data_loader.py

Blame
  • spejd_service.cpp NaN GiB
    #include "spejd_service.hpp"
    
    spejd *SpejdWorker::engine=NULL;
    
    bool copyFile(const char *SRC, const char* DEST)
    {
        std::ifstream src(SRC, std::ios::binary);
        std::ofstream dest(DEST, std::ios::binary);
        dest << src.rdbuf();
        return src && dest;
    }
    
    bool CopyDir( boost::filesystem::path const & source,
                  boost::filesystem::path const & destination )
    {
        try
        {
            if( !boost::filesystem::exists(source) ||
                !boost::filesystem::is_directory(source) )
            {
                std::cerr << "Source directory "
                          << source.string()
                          << " does not exist or is not a directory."
                          << '\n';
                return false;
            }
             
            if( boost::filesystem::exists( destination ) )
            {
                std::cerr << "Destination directory "
                          << destination.string()
                          << " already exists." << '\n';
                return false;
            }
     
            if( !boost::filesystem::create_directory( destination ) )
            {
                std::cerr << "Unable to create destination directory"
                          << destination.string() << '\n';
                return false;
            }
        }
     
        catch( boost::filesystem::filesystem_error const & e)
        {
            std::cerr << e.what() << '\n';
            return false;
        }
     
        // Iterate through the source directory
        for( boost::filesystem::directory_iterator file( source );
             file != boost::filesystem::directory_iterator(); 
             ++file )
        {
            try
            {
                boost::filesystem::path current( file->path() );
                if( boost::filesystem::is_directory( current ) )
                {
                    // Found directory: Recursion
                    if( !CopyDir( current, destination / current.filename() ) )
                    {
                        return false;
                    }
                }
                else
                {
                    // Found file: Copy
                    //boost::filesystem::copy_file( current,destination / current.filename() );
    				copyFile(current.string().c_str(),(destination / current.filename()).string().c_str());
                }
            } 
     
            catch( boost::filesystem::filesystem_error const & e )
            {
                std:: cerr << e.what() << '\n';
            }
        }
        return true;
    }
     
    void SpejdWorker::process(std::string task_path, boost::property_tree::ptree &options, std::string output_path) 
    {   
    	std::vector<std::string> filelist;
    	CopyDir(task_path,output_path);
    	filelist.push_back(output_path+"/ann_morphosyntax.xml.gz");
    	engine->process_files(filelist); 
    }
    
    void SpejdWorker::static_init(boost::property_tree::ptree config)
    {
        spejd::set_out_stream(&std::cout);
    	std::string configname = config.get<std::string>("tool.model");
        std::cout << "Hello!" << std::endl;
    	engine = spejd::create_instance();
        std::cout << "Loading model" << std::endl;
    	engine->load_config_file(std::string(configname));
        std::cout << "Model loaded" << std::endl;
        engine->commit_config();
    }
    
    void SpejdWorker::process(ptree &data, ptree &options) { 
    } 
    
    int main(int argv, char* argc[])
    {
        run_workers<SpejdWorker>(argv,argc);
    }