[TOC]

如何通过Gitlab的API接口获取远程仓库中的文件内容

在一些实际情况中,希望能够直接像读取本地文件一样读取远程仓库中的文件内容,避免git操作失败的情况下读取的本地缓存的文件内容。由于项目使用gitLab管理配置文件,查询了GitLabApi,其提供了诸多API接口,包括常见的git操作、项目管理以及我们需要的获取文件内容等接口。

GitLab获取仓库中文件内容的API文档(https://docs.gitlab.com/ee/api/repository_files.html#get-file-from-repository)

格式说明:

1
2
3
4
5
# 请求格式
GET /projects/:id/repository/files/:file_path

# 请求实例
curl --request GET --header 'PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK' 'https://gitlab.example.com/api/v4/projects/{项目ID}/repository/files/app%2Fmodels%2Fkey%2Erb?ref=master'

通过分析可以发现,如果想获取仓库中文件内容,需要以下几个要素:

仓库地址
项目id
用户的private token
经过url编码的文件全路径
文件所在的分支

private static String GITLAB_FILECONTENT_API = “http://#{REPO_IP}/api/v3/projects/#{PROJECT_ID}/repository/files?private_token=#{PRIVATE_TOKEN}&file_path=#{FILE_PATH}&ref=#{BRANCH_NAME}”;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
// 获取用户的private token
private static String GITLAB_SESSION_API = "http://#{REPO_IP}/api/v3/session?login=#{USER_NAME}&password=#{PASSWORD}";
/**
* 根据用户名称和密码获取gitlab的private token,为Post请求
* 注意,这个接口时POST请求,正确的HTTP响应码是201,而非200.
* @param ip gitlab仓库的ip
* @param userName 登陆gitlab的用户名
* @param password 登陆gitlab的密码
* @return 返回该用户的private token
*/
public static String getPrivateTokenByPassword(String ip, String userName, String password) {
/** 1.参数替换,生成获取指定用户privateToken地址 */
// 校验参数
Objects.requireNonNull(ip, "参数ip不能为空!");
Objects.requireNonNull(userName, "参数userName不能为空!");
Objects.requireNonNull(password, "参数password不能为空!");
// 参数准备,存入map
Map<String, String> params = new HashMap<String, String>(4);
params.put("REPO_IP", ip);
params.put("USER_NAME", userName);
params.put("PASSWORD", password);
// 调用工具类替换,得到具体的调用地址
String reqUserTokenUrl = PlaceholderUtil.anotherReplace(GITLAB_SESSION_API, params);
sysLogger.debug(String.format("获取用户:%s的private token地址为:%s", userName,reqUserTokenUrl));

/** 2.访问url,获取指定用户的信息 */
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> response = restTemplate.postForEntity(reqUserTokenUrl, null,String.class);
sysLogger.debug(String.format("响应头为:%s,响应体为:%s", response.getHeaders(), response.getBody()));

/** 3.解析结果 */
String body = response.getBody();
JSONObject jsonBody = JsonUtil.parseObjectToJSONObject(body);
String privateToken =jsonBody.getString("private_token");
sysLogger.debug(String.format("获取到用户:%s的privateToken为:%s", userName, privateToken));

/** 4.返回privateToken */
return privateToken;
}


// 获取项目的projectId
private static String GITLAB_SINGLE_PROJECT_API = "http://#{REPO_IP}/api/v3/projects/#{PROJECT_PATH}?private_token=#{PRIVATE_TOKEN}";
/**
* 使用gitLab api获取指定项目的projectId,为Get请求
*
* @param ip 项目仓库的ip地址
* @param projectPath 项目的path,如:http://192.168.59.185/acountting/dispatcher-cloud.git,则projectPath为:acountting/dispatcher-cloud
* @param privateToken 用户个人访问gitlab库时的privateToken,可以通过{@link GitLabAPIUtils#getPrivateTokenByPassword}获取
* @return 返回目的projectId
*/
public static String getProjectId(String ip, String projectPath, String privateToken) {
/** 1.参数替换,生成访问获取project信息的uri地址 */
// 校验参数
Objects.requireNonNull(ip, "参数ip不能为空!");
Objects.requireNonNull(projectPath, "参数projectPath不能为空!");
Objects.requireNonNull(privateToken, "参数privateToken不能为空!");
// 参数准备,存入map
Map<String, String> params = new HashMap<String, String>(4);
params.put("REPO_IP", ip);
params.put("PRIVATE_TOKEN", privateToken);
// gitlab api要求项目的path需要安装uri编码格式进行编码,比如"/"编码为"%2F"
try {
params.put("PROJECT_PATH", URLEncoder.encode(projectPath, "UTF-8"));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(String.format("对%s进行URI编码出错!", projectPath));
}
// 调用工具类替换,得到具体的调用地址
String getSingleProjectUrl = PlaceholderUtil.anotherReplace(GITLAB_SINGLE_PROJECT_API, params);
sysLogger.debug(String.format("获取projectId的url:%s", getSingleProjectUrl));

// 创建URI对象
URI url = null;
try {
url = new URI(getSingleProjectUrl);
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("使用%s创建URI出错!", getSingleProjectUrl));
}

/** 2.访问url,获取制定project的信息 */
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> reslut = restTemplate.getForEntity(url, String.class);
sysLogger.debug(String.format("响应头为:%s,响应体为:%s", reslut.getHeaders(), reslut.getBody()));

/** 3.解析结果 */
if (reslut.getStatusCode() != HttpStatus.OK ) {
throw new RuntimeException(String.format("请求%s出错!错误码为:%s", url, reslut.getStatusCode()));
}
// 如果响应码是200,说明正常拿到响应结果,解析出projectId返回即可
JSONObject responBody = JsonUtil.parseObjectToJSONObject(reslut.getBody());
String projectRepo = responBody.getString("http_url_to_repo");
String projectId = responBody.getString("id");
sysLogger.info(String.format("获取到项目:%s的projectId为:%s", projectRepo, projectId));

/** 4.返回projectId */
return projectId;
}


// 获取仓库文件内容
// 特别需要注意的是返回的文件内容是base64编码,拿到结果后需要解码才能获取原始内容。
private static String GITLAB_FILECONTENT_API = "http://#{REPO_IP}/api/v3/projects/#{PROJECT_ID}/repository/files?private_token=#{PRIVATE_TOKEN}&file_path=#{FILE_PATH}&ref=#{BRANCH_NAME}";
public static String getFileContentFromRepository(String ip,String projectPath,String userName,String password,String fileFullPath, String branchName) throws Exception {
// 校验参数
Objects.requireNonNull(ip, "参数ip不能为空!");
Objects.requireNonNull(projectPath, "参数projectPath不能为空!");
Objects.requireNonNull(userName, "参数userName不能为空!");
Objects.requireNonNull(password, "参数password不能为空!");
Objects.requireNonNull(fileFullPath, "参数fileFullPath不能为空!");
Objects.requireNonNull(branchName, "参数branchName不能为空!");

/** 1.依据用户名、密码获取到用户的privateToken */
String privateToken = getPrivateTokenByPassword(ip, userName, password);

/** 2.使用privateToken获取项目的projectId */
String projectId = getProjectId(ip, projectPath, privateToken);

/** 3.使用参数替换形成请求git库中文件内容的uri */
// 参数准备,存入map
Map<String, String> params = new HashMap<String, String>(4);
params.put("REPO_IP", ip);
params.put("PROJECT_ID", projectId);
params.put("PRIVATE_TOKEN", privateToken);
params.put("FILE_PATH", fileFullPath);
params.put("BRANCH_NAME", branchName);
// 使用工具类替换参数
String reqFileCotnetUri = PlaceholderUtil.anotherReplace(GITLAB_FILECONTENT_API, params);
sysLogger.debug(String.format("获取文件:%s的uri:%s", fileFullPath, reqFileCotnetUri));

/** 4.请求gitlab获取文件内容 */
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> response = restTemplate.getForEntity(reqFileCotnetUri, String.class);
sysLogger.debug(String.format("响应头为:%s,响应体为:%s", response.getHeaders(), response.getBody()));

/** 5.解析响应结果内容 */
String body = response.getBody();
JSONObject jsonBody = JsonUtil.parseObjectToJSONObject(body);
String fileName = jsonBody.getString("file_name");
String filePath = jsonBody.getString("file_path");
String encoding = jsonBody.getString("encoding");
String content = jsonBody.getString("content");
String commitId = jsonBody.getString("commit_id");
String lastCommitId = jsonBody.getString("last_commit_id");

// 内容已经base64编码,如果需要获取原始文件内容可以参看api:https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository
content = new String(Base64.decode(content), "UTF-8");

sysLogger.debug(String.format(
"获取http://%s 上%s项目 %s分支的%s文件,响应信息是:fileName :%s ,filePath:%s , 编码:%s ,内容:%s , commitId:%s ,lastCommitId :%s",
ip,projectPath, branchName, fileFullPath, fileName, filePath, encoding, content, commitId,
lastCommitId));

/** 6.返回指定文件的内容 */
sysLogger.debug(String.format("解析得到文件内容为:%s", content));
return content;
}

// 此外,考虑到项目中实际获取的是配置文件内容,为了剔除不必要的空行、注释行,提供了工具类方法对解码的原始文件内容进行处理:
public static String getCleanFileContentFromRepository(String ip,String projectPath,String userName,String password,String fileFullPath, String branchName) throws Exception{

/** 1.获取到原始文件内容 */
String fileContent = getFileContentFromRepository(ip, projectPath, userName, password, fileFullPath, branchName);
if (fileContent.isEmpty()) {
return fileContent;
}

/** 2.所有行转换为list,并过滤掉空行、#开头的注释行 */
List<String> list = Arrays.asList(fileContent.split("\n")).stream().filter(a-> (!a.trim().isEmpty()&&!a.trim().startsWith("#"))).collect(Collectors.toList());

/** 3.转为一整行字符串返回 */
StringBuilder sb = new StringBuilder();
int size =list.size();
for (int i = 0; i < size; i++) {
sb.append(list.get(i).trim());
}
return sb.toString();
}

需要特别注意,在方法中对配置好的project path进行url编码后,没有直接使用RestTemplate创建get请求获取项目信息,因为实践中发现会出现将本义编码好的如:src%2FHelloWorld.java变为:src%256FHelloWorld.java,具体没有深入RestTemplate源码,所以直接创建URI对象,避免这种情况出现。

5.1 替换参数工具类

对#{}包裹的参数进行替换,代码如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
public class PlaceholderUtil {
/** 默认替换形如#{param}的占位符 */
private static Pattern pattern = Pattern.compile("\\#\\{.*?\\}");

/**
* 替换字符串中形如#{}的占位符
* @param src
* @param parameters
* @return
*/
public static String replace(String src, Map<String, Object> parameters) {
Matcher paraMatcher = pattern.matcher(src);

// 存储参数名
String paraName = "";
String result = new String(src);

while (paraMatcher.find()) {
paraName = paraMatcher.group().replaceAll("\\#\\{", "").replaceAll("\\}", "");
Object objParam = parameters.get(paraName);
if(objParam!=null){
result = result.replace(paraMatcher.group(), objParam.toString());
}
}
return result;
}


/**
* 替换字符串中形如#{}的占位符
* @param src
* @param parameters
* @return
*/
public static String anotherReplace(String str, Map<String, String> params) {
Map<String, Object> newParams = new HashMap<>(params);
return replace(str, newParams);
}

}

5.2 测试类:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
public class GitLabAPIUtilsTest {

String repoIp;
String privateToken;
String projectPath1;
String projectPath2;


String userName;
String password;

String fileFullPath;
String branchName;

@Before
public void setUp() throws Exception {
repoIp = "gitlab仓库ip";
projectPath1 = "acountting/accounting-config-repo";
projectPath2 = "acountting/csv-filefront-cloud";

userName = "用户名";
password="密码";

fileFullPath = "/apps/cmup-clearing/0055-account.config";
branchName = "develop";
}

@After
public void tearDown() throws Exception {
repoIp = null;
projectPath1 = null;
projectPath2 = null;
userName = null;
password = null;
fileFullPath = null;
branchName = null;
}

@Test
public void testGetProjectId() {
String privateToken = GitLabAPIUtils.getPrivateTokenByPassword(repoIp, userName, password);
String projectId = GitLabAPIUtils.getProjectId(repoIp, projectPath1, privateToken);
System.out.println("projectId = " + projectId);
}

@Test
public void testGetPrivateToken() {
String privateToken = GitLabAPIUtils.getPrivateTokenByPassword(repoIp, userName, password);
System.out.println("projectId = " + privateToken);
}

@Test
public void testGetFileContent() {
String fileContent;
try {
fileContent = GitLabAPIUtils.getCleanFileContentFromRepository(repoIp, projectPath1, userName, password, fileFullPath, branchName);

System.out.println("fileContent = " + fileContent);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

}

https://www.jianshu.com/p/0d6da25b1ab6