gitlab项目爬取下载

java

浏览数:1,097

2019-1-12

片段 1片段 2


gitlab

package com.test.gitlab;import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/***
 * @Author maochuang.li
 *
 * @Email itdreamlmc@163.com
 *
 * @Date Create in 9:46 2018/2/28 0028
 */
public class DownloadGitlab {

    /**
     * gitlab 地址
     */
    private static final String baseUrl = "http://192.168.12.99:9000";

    /**
     * 文件保存路径
     */
    private static final String saveFilePath = "C:\\fuyinhy-back";    public static void main(String[] args) throws Exception {
        getFileBranchList(getAllActiveUser());
    }

    /**
     * 单线程下载项目
     *
     * @param gitlabUsers
     * @Author maochuang.li
     * @Date Create in 14:00 2018/3/1 0001
     */
    public static void getFileBranchList(List<String> gitlabUsers) throws Exception {
        for (String userName : gitlabUsers) {
            String url = baseUrl + "/u/";
            String data = HttpUtils.doGet(url + userName);
            String patt = "<a class=\"project\" href=\"(.*?)\">";
            //获取项目名称
            List<String> prjNames = regexGetText(data, patt);
            //循环遍历获取分支名称
            if (prjNames != null && !"".equals(prjNames)) {
                for (String name : prjNames) {
                    data = HttpUtils.doGet(baseUrl + name + "/commits/master");
                    //获取所有分支名
                    patt = "<option value=\".*?\">(.*?)</option>";
                    List<String> branchNames = regexGetText(data, patt);
                    downLoadZip(name, branchNames, userName);
                }
            }
        }
    }

    /**
     * @param
     * @Author maochuang.li
     * @Date Create in 11:37 2018/3/1 0001
     */
    public static List<String> getAllActiveUser() {
        String url = baseUrl + "/autocomplete/users.json?search=&per_page=20&active=true&project_id=9&current_user=true";
        String resultData = HttpUtils.doGet(url);
        String regex = "username\":\"(.*?)\",\"";
        List<String> gitlabUsers = regexGetText(resultData, regex);
        return gitlabUsers;
    }

    /**
     * 下载压缩包
     *
     * @param prjName     项目名称
     * @param branchNames 项目分支列表
     * @param userName    用户名
     * @Author maochuang.li
     * @Date Create in 10:32 2018/2/28 0028
     */
    private static void downLoadZip(String prjName, List<String> branchNames, String userName) throws Exception {
        String projectName = prjName.substring(prjName.lastIndexOf("/") + 1);
        File file = new File(saveFilePath + File.separator + userName + File.separator + projectName);
        if (!file.exists())
            file.mkdirs();
        if (branchNames != null && branchNames.size() > 0) {
            for (String branchName : branchNames) {
                System.out.println("正在下载:项目名称->" + prjName + " 分支名称->" + branchName);
                String url = baseUrl + prjName + "/repository/archive.zip?ref=" + branchName;
                InputStream inputStream = new URL(url).openStream();
                byte[] buf = new byte[4048];
                int len = -1;
                FileOutputStream fileOutputStream = new FileOutputStream(new File(file, projectName + "-" + branchName + ".zip"));
                while ((len = inputStream.read(buf, 0, buf.length)) != -1) {
                    fileOutputStream.write(buf, 0, len);
                    fileOutputStream.flush();
                }
                fileOutputStream.close();
                System.out.println("完成下载:项目名称->" + prjName + " 分支名称->" + branchName);
            }
        }
    }

    /**
     * 正则文本匹配
     *
     * @param srl
     * @param patt
     * @Author maochuang.li
     * @Date Create in 10:30 2018/2/28 0028
     */
    private static List<String> regexGetText(String srl, String patt) {
        ArrayList<String> dataList = new ArrayList<>();
        Pattern r = Pattern.compile(patt);
        Matcher m = r.matcher(srl);
        while (m.find()) {
            dataList.add(m.group(1));
        }
        return dataList;
    }
}


http工具类

package com.test.gitlab;

import com.aliyun.oss.common.utils.HttpUtil;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.GeneralSecurityException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;

/**
 * HTTP 请求工具类
 *
 * @author : lmc
 * @version : 1.0.0
 * @date : 2017/6/25
 * @see : TODO
 */
public class HttpUtils {
    private static PoolingHttpClientConnectionManager connMgr;
    private static RequestConfig requestConfig;
    private static final int MAX_TIMEOUT = 60000;

    /**
     * 日志
     **/
    protected static final Logger logger = LoggerFactory.getLogger(HttpUtil.class);

    static {
        // 设置连接池
        connMgr = new PoolingHttpClientConnectionManager();
        // 设置连接池大小
        connMgr.setMaxTotal(100);
        connMgr.setDefaultMaxPerRoute(connMgr.getMaxTotal());

        RequestConfig.Builder configBuilder = RequestConfig.custom();
        // 设置连接超时
        configBuilder.setConnectTimeout(MAX_TIMEOUT);
        // 设置读取超时
        configBuilder.setSocketTimeout(MAX_TIMEOUT);
        // 设置从连接池获取连接实例的超时
        configBuilder.setConnectionRequestTimeout(MAX_TIMEOUT);
        // 在提交请求之前 测试连接是否可用
        configBuilder.setStaleConnectionCheckEnabled(true);
        requestConfig = configBuilder.build();
    }

    /**
     * 发送 GET 请求(HTTP),不带输入数据
     *
     * @param url
     * @return
     */
    public static String doGet(String url) {
        return doGet(url, new HashMap<String, Object>());
    }

    /**
     * 发送 GET 请求(HTTP),K-V形式
     *
     * @param url
     * @param params
     * @return
     */
    public static String doGet(String url, Map<String, Object> params) {
        String apiUrl = url;
        StringBuffer param = new StringBuffer();
        int i = 0;
        for (String key : params.keySet()) {
            if (i == 0)
                param.append("?");
            else
                param.append("&");
            param.append(key).append("=").append(params.get(key));
            i++;
        }
        apiUrl += param;
        String result = null;
        HttpClient httpclient = new DefaultHttpClient();
        try {
            HttpGet httpPost = new HttpGet(apiUrl);
            HttpResponse response = httpclient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            result = EntityUtils.toString(entity, "UTF-8");
        } catch (IOException e) {
            logger.error(apiUrl, e);
        }
        logger.info("url:" + apiUrl + "-----result:" + result);
        return result;
    }    /**
     * 发送 GET 请求(HTTP),K-V形式
     *
     * @param url
     * @return
     */
    public static byte[] doGetReturnByte(String url,File filePathName) {
        String apiUrl = url;
        String result = null;
        HttpClient httpclient = new DefaultHttpClient();
        try {
            HttpGet httpPost = new HttpGet(apiUrl);
            HttpResponse response = httpclient.execute(httpPost);

            HttpEntity entity = response.getEntity();
            byte[] bytes = EntityUtils.toByteArray(entity);
            return bytes;
        } catch (IOException e) {
            logger.error(apiUrl, e);
        }
        logger.info("url:" + apiUrl + "-----result:" + result);
        return null;
    }    /**
     * 发送 POST 请求(HTTP),不带输入数据
     *
     * @param apiUrl
     * @return
     */
    public static String doPost(String apiUrl) {
        return doPost(apiUrl, new HashMap<String, Object>());
    }

    /**
     * 发送 POST 请求(HTTP),K-V形式
     *
     * @param apiUrl API接口URL
     * @param params 参数map
     * @return
     */
    public static String doPost(String apiUrl, Map<String, Object> params) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        String httpStr = null;
        HttpPost httpPost = new HttpPost(apiUrl);
        CloseableHttpResponse response = null;

        try {
            httpPost.setConfig(requestConfig);
            List<NameValuePair> pairList = new ArrayList<>(params.size());
            for (Map.Entry<String, Object> entry : params.entrySet()) {
                NameValuePair pair = new BasicNameValuePair(entry.getKey(), entry
                        .getValue().toString());
                pairList.add(pair);
            }
            logger.info("【post parameters】:{}", pairList);
            httpPost.setEntity(new UrlEncodedFormEntity(pairList, Charset.forName("UTF-8")));
            response = httpClient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            httpStr = EntityUtils.toString(entity, "UTF-8");
        } catch (IOException e) {
            logger.error(httpPost.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(apiUrl, e);
                }
            }
        }
        logger.debug("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * @param apiUrl
     * @param param
     * @param header 自定义header
     * @return
     */
    public static String doPost(String apiUrl, String param, Header header) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        String httpStr = null;
        HttpPost httpPost = new HttpPost(apiUrl);
        CloseableHttpResponse response = null;
        try {
            if (null != header) {
                httpPost.setHeader(header);
            }
            httpPost.setConfig(requestConfig);
            StringEntity stringEntity = new StringEntity(param, "UTF-8");//解决中文乱码问题
            stringEntity.setContentEncoding("UTF-8");
            httpPost.setEntity(stringEntity);
            response = httpClient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            httpStr = EntityUtils.toString(entity, "UTF-8");
        } catch (IOException e) {
            logger.error(httpPost.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(httpPost.toString(), e);
                }
            }
        }
        logger.info("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * 发送 POST 请求(HTTP),JSON形式
     *
     * @param apiUrl
     * @param
     * @return
     */
    public static String doPost(String apiUrl, String param) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        String httpStr = null;
        HttpPost httpPost = new HttpPost(apiUrl);
        CloseableHttpResponse response = null;
        try {
            httpPost.setConfig(requestConfig);
            StringEntity stringEntity = new StringEntity(param, "UTF-8");//解决中文乱码问题
            stringEntity.setContentEncoding("UTF-8");
            httpPost.setEntity(stringEntity);
            response = httpClient.execute(httpPost);
            HttpEntity entity = response.getEntity();
            httpStr = EntityUtils.toString(entity, "UTF-8");
        } catch (IOException e) {
            logger.error(httpPost.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(httpPost.toString(), e);
                }
            }
        }
        logger.info("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * 发送 SSL POST 请求(HTTPS),K-V形式
     *
     * @param apiUrl API接口URL
     * @param params 参数map
     * @return
     */
    public static String doPostSSL(String apiUrl, Map<String, Object> params) {
        CloseableHttpClient httpClient = HttpClients.custom().setSSLSocketFactory(createSSLConnSocketFactory()).setConnectionManager(connMgr).setDefaultRequestConfig(requestConfig).build();
        HttpPost httpPost = new HttpPost(apiUrl);
        CloseableHttpResponse response = null;
        String httpStr = null;
        try {
            httpPost.setConfig(requestConfig);
            List<NameValuePair> pairList = new ArrayList<NameValuePair>(params.size());
            for (Map.Entry<String, Object> entry : params.entrySet()) {
                NameValuePair pair = new BasicNameValuePair(entry.getKey(), entry
                        .getValue().toString());
                pairList.add(pair);
            }
            httpPost.setEntity(new UrlEncodedFormEntity(pairList, Charset.forName("utf-8")));
            response = httpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode != HttpStatus.SC_OK) {
                return null;
            }
            HttpEntity entity = response.getEntity();
            if (entity == null) {
                return null;
            }
            httpStr = EntityUtils.toString(entity, "utf-8");
        } catch (Exception e) {
            logger.error(httpPost.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(httpPost.toString(), e);
                }
            }
        }
        logger.info("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * 发送 SSL POST 请求(HTTPS),JSON形式
     *
     * @param apiUrl API接口URL
     * @param
     * @return
     */
    public static String doPostSSL(String apiUrl, String param) {
        CloseableHttpClient httpClient = HttpClients.custom().setSSLSocketFactory(createSSLConnSocketFactory()).setConnectionManager(connMgr).setDefaultRequestConfig(requestConfig).build();
        HttpPost httpPost = new HttpPost(apiUrl);
        CloseableHttpResponse response = null;
        String httpStr = null;
        try {
            httpPost.setConfig(requestConfig);
            StringEntity stringEntity = new StringEntity(param, "UTF-8");//解决中文乱码问题
            stringEntity.setContentEncoding("UTF-8");
            httpPost.setEntity(stringEntity);
            response = httpClient.execute(httpPost);
            int statusCode = response.getStatusLine().getStatusCode();
            if (statusCode != HttpStatus.SC_OK) {
                return null;
            }
            HttpEntity entity = response.getEntity();
            if (entity == null) {
                return null;
            }
            httpStr = EntityUtils.toString(entity, "utf-8");
        } catch (Exception e) {
            logger.error(httpPost.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(httpPost.toString(), e);
                }
            }
        }
        logger.info("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * 创建SSL安全连接
     *
     * @return
     */
    private static SSLConnectionSocketFactory createSSLConnSocketFactory() {
        SSLConnectionSocketFactory sslsf = null;
        try {
            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                    return true;
                }
            }).build();
            sslsf = new SSLConnectionSocketFactory(sslContext, new X509HostnameVerifier() {

                @Override
                public boolean verify(String arg0, SSLSession arg1) {
                    return true;
                }

                @Override
                public void verify(String host, SSLSocket ssl) throws IOException {
                }

                @Override
                public void verify(String host, X509Certificate cert) throws SSLException {
                }

                @Override
                public void verify(String host, String[] cns, String[] subjectAlts) throws SSLException {
                }
            });
        } catch (GeneralSecurityException e) {
            logger.error("ssl", e);
        }
        return sslsf;
    }

    /***
     *发送POST请求 文件上传
     * @param apiUrl 上传地址
     * @param fileNamePath 文件绝对路径
     * @param paramsMap 上传其他参数
     * @return
     * @throws Exception
     */
    public static String doPostFileUpload(String apiUrl, File fileNamePath, Map<String, Object> paramsMap) throws Exception {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        HttpPost post = new HttpPost(apiUrl);
        String httpStr = null;
        HttpResponse response = null;
        try {
            FileBody fileBody = new FileBody(fileNamePath);
            MultipartEntityBuilder builder = MultipartEntityBuilder.create();
            builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
            builder.addPart("file", fileBody);
            Set<Map.Entry<String, Object>> entries = paramsMap.entrySet();
            for (Map.Entry<String, Object> entry : entries) {
                builder.addPart(entry.getKey(), new StringBody(entry.getValue() != null ? entry.getValue().toString() : ""));
            }
            HttpEntity entity = builder.build();
            post.setEntity(entity);
            response = httpclient.execute(post);
            httpStr = EntityUtils.toString(response.getEntity(), "UTF-8");
        } catch (IOException e) {
            logger.error(post.toString(), e);
        } catch (ParseException e) {
            logger.error(post.toString(), e);
        } finally {
            if (response != null) {
                try {
                    EntityUtils.consume(response.getEntity());
                } catch (IOException e) {
                    logger.error(post.toString(), e);
                }
            }
        }
        logger.info("url:" + apiUrl + "-----result:" + httpStr);
        return httpStr;
    }

    /**
     * 测试方法
     *
     * @param args
     */
    public static void main(String[] args) throws Exception {
        String url = "http://192.168.23.121:18112/add_mlearn";
        String fileName = "C:\\Users\\lmc\\Desktop\\test\\test.java";
        HashMap<String, Object> map = new HashMap<>();
        map.put("filename", "fileName");
        map.put("mlname", "测试添加");
        String s = doPostFileUpload(url, new File(fileName), map);
        System.out.println("上传文件结果:" + s);
    }
}