kbruen/gaulthiergain-tools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

code reorganisation

Browse source
This commit is contained in:
Rob1103 2023-03-15 11:29:53 +01:00
parent b5ec3c3329
commit a4ee7e7526
7 changed files with 475 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
srcs/buildtool/args.go
View file

@ -20,6 +20,7 @@ const (
objsArg = "objects"
makefileArg = "makefile"
configArg = "config"
interdependArg = "interdepend"
)
// ParseArguments parses arguments of the application.
@ -43,6 +44,10 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
"for Makefile"})
args.InitArgParse(p, args, u.STRINGLIST, "c", configArg,
&argparse.Options{Required: false, Help: "Add configuration files"})
args.InitArgParse(p, args, u.BOOL, "i", interdependArg,
&argparse.Options{Required: false, Default: false,
Help: "Use the source files filtered by the deptool interdependence graph to build " +
"the app"})
return u.ParserWrapper(p, os.Args)
}

13
srcs/buildtool/microlibs_process.go
View file

@ -133,8 +133,8 @@ func fetchSymbolsExternalLibs(folder string,
return externalLibs, nil
}
// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static and
// dynamic analyses together into a map structure.
// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static,
// dynamic and source files analyses together into a map structure.
//
// It returns the map containing all the symbols and system calls.
func putJsonSymbolsTogether(data *u.Data) map[string]string {
@ -156,6 +156,14 @@ func putJsonSymbolsTogether(data *u.Data) map[string]string {
dataMap[k] = ""
}
for k, v := range data.SourcesData.Symbols {
dataMap[k] = v
}
for k := range data.SourcesData.SystemCalls {
dataMap[k] = ""
}
return dataMap
}
@ -219,6 +227,7 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string,
// Perform the symbol matching
matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols)
//matchedLibs = append(matchedLibs, LWIP)
return matchedLibs, externalLibs, nil
}

9
srcs/common/data.go
View file

@ -6,10 +6,11 @@
package common
// Exported struct that represents static and dynamic data.
// Exported struct that represents static, dynamic and sources data.
type Data struct {
StaticData StaticData `json:"static_data"`
DynamicData DynamicData `json:"dynamic_data"`
SourcesData SourcesData `json:"sources_data"`
}
// Exported struct that represents data for static dependency analysis.
@ -26,3 +27,9 @@ type DynamicData struct {
SystemCalls map[string]int `json:"system_calls"`
Symbols map[string]string `json:"symbols"`
}
// Exported struct that represents data for sources dependency analysis.
type SourcesData struct {
SystemCalls map[string]int `json:"system_calls"`
Symbols map[string]string `json:"symbols"`
}

7
srcs/dependtool/args.go
View file

@ -23,7 +23,6 @@ const (
fullDepsArg = "fullDeps"
fullStaticAnalysis = "fullStaticAnalysis"
typeAnalysis = "typeAnalysis"
interdependArg = "interdepend"
)
// parseLocalArguments parses arguments of the application.
@ -54,10 +53,8 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
Help: "Full static analysis (analyse shared libraries too)"})
args.InitArgParse(p, args, u.INT, "", typeAnalysis,
&argparse.Options{Required: false, Default: 0,
Help: "Kind of analysis (0: both; 1: static; 2: dynamic)"})
args.InitArgParse(p, args, u.BOOL, "i", interdependArg,
&argparse.Options{Required: false, Default: false,
Help: "Create the source files interdependence graph"})
Help: "Kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: " +
"sources; 5: stripped-down app and json for buildtool)"})
return u.ParserWrapper(p, os.Args)
}

263
srcs/dependtool/interdependence_graph.go Normal file
View file

@ -0,0 +1,263 @@
package dependtool
import (
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
u "tools/srcs/common"
)
// ---------------------------------Gather Data---------------------------------
// getProgramFolder gets the folder path in which the given program is located, according to the
// Unikraft standard (e.g., /home/.../apps/programFolder/.../program).
//
// It returns the folder containing the program files according to the standard described above.
func getProgramFolder(programPath string) string {
tmp := strings.Split(programPath, u.SEP)
i := 2
for ; i < len(tmp); i++ {
if tmp[len(tmp)-i] == "apps" {
break
}
}
folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP)
return folderPath
}
// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file.
//
// It returns a slice containing the found header names.
func sourceFileIncludesAnalysis(sourceFile string) []string {
var fileIncludes []string
fileLines, err := u.ReadLinesFile(sourceFile)
if err != nil {
u.PrintErr(err)
}
for _, line := range fileLines {
if strings.Contains(line, "#include") {
line = strings.ReplaceAll(line, " ", "")
line = strings.Split(line, "#include")[1]
if strings.HasPrefix(line, "\"") {
fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1])
} else if strings.HasPrefix(line, "<") {
fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1])
}
}
}
return fileIncludes
}
// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using
// the gcc preprocessor.
//
// It returns a slice containing the found header names.
func gccSourceFileIncludesAnalysis(sourceFile, outAppFolder string) ([]string, error) {
var fileIncludes []string
// gcc command
outputStr, err := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", outAppFolder})
// if gcc command returns an error, prune file: it contains non-standard libs
if err != nil {
return make([]string, 0), err
}
outputSlice := strings.Split(outputStr, "\n")
for _, line := range outputSlice {
// Only interested in headers not coming from the standard library
if strings.Contains(line, "\""+outAppFolder) {
line = strings.Split(line, "\""+outAppFolder)[1]
includeDirective := line[0:strings.Index(line[0:], "\"")]
if !u.Contains(fileIncludes, includeDirective) {
fileIncludes = append(fileIncludes, includeDirective)
}
}
}
return fileIncludes, nil
}
// pruneRemovableFiles prunes interdependence graph elements if the latter are unused header files.
func pruneRemovableFiles(interdependMap *map[string][]string) {
for internalFile := range *interdependMap {
// No removal of C/C++ source files
if filepath.Ext(internalFile) != ".c" && filepath.Ext(internalFile) != ".cpp" &&
filepath.Ext(internalFile) != ".cc" {
// Lookup for files depending on the current header file
depends := false
for _, dependencies := range *interdependMap {
for _, dependency := range dependencies {
if internalFile == dependency {
depends = true
break
}
}
if depends {
break
}
}
// Prune header file if unused
if !depends {
delete(*interdependMap, internalFile)
}
}
}
}
// pruneElemFiles prunes interdependence graph elements if the latter contain the substring in
// argument.
func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) {
// Lookup for key elements containing the substring and prune them
for internalFile := range *interdependMap {
if strings.Contains(internalFile, pruneElem) {
delete(*interdependMap, internalFile)
// Lookup for key elements that depend on the key found above and prune them
for file, dependencies := range *interdependMap {
for _, dependency := range dependencies {
if dependency == internalFile {
pruneElemFiles(interdependMap, file)
}
}
}
}
}
}
// requestUnikraftExtLibs collects all the GitHub repositories of Unikraft through the GitHub API
// and returns the whole list of Unikraft external libraries.
func requestUnikraftExtLibs() []string {
var extLibsList, appsList []string
// Only 2 Web pages of repos as for february 2023 (125 repos - 100 repos per page)
nbPages := 2
for i := 1; i <= nbPages; i++ {
// HTTP Get request
resp, err := http.Get("https://api.github.com/orgs/unikraft/repos?page=" +
strconv.Itoa(i) + "&per_page=100")
if err != nil {
u.PrintErr(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
u.PrintErr(err)
}
// Collect libs
fileLines := strings.Split(string(body), "\"name\":\"lib-")
for i := 1; i < len(fileLines); i++ {
extLibsList = append(extLibsList, fileLines[i][0:strings.Index(fileLines[i][0:],
"\"")])
}
// Collect apps
fileLines = strings.Split(string(body), "\"name\":\"app-")
for i := 1; i < len(fileLines); i++ {
appsList = append(appsList, fileLines[i][0:strings.Index(fileLines[i][0:],
"\"")])
}
}
// Avoid libs that are also apps (e.g. nginx, redis)
for i, lib := range extLibsList {
if u.Contains(appsList, lib) {
extLibsList = append(extLibsList[:i], extLibsList[i+1:]...)
}
}
return extLibsList
}
// -------------------------------------Run-------------------------------------
// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each
// C/C++ source file of a program and builds an interdependence graph (dot file) between all these
// source files.
func interdependAnalyser(programPath, programName, outFolder string) string {
// Find all program source files
sourceFiles, err := findSourcesFiles(getProgramFolder(programPath))
if err != nil {
u.PrintErr(err)
}
// Create a folder and copy all source files into it for use with the gcc preprocessor
//tmp := strings.Split(getProgramFolder(programPath), u.SEP)
//outAppFolder := strings.Join(tmp[:len(tmp)-1], u.SEP) + u.SEP + programName +
//"_deptool_output" + u.SEP
outAppFolder := outFolder + programName + u.SEP
_, err = u.CreateFolder(outAppFolder)
if err != nil {
u.PrintErr(err)
}
var outAppFiles []string
for _, sourceFilePath := range sourceFiles {
if err := u.CopyFileContents(sourceFilePath,
outAppFolder+filepath.Base(sourceFilePath)); err != nil {
u.PrintErr(err)
}
outAppFiles = append(outAppFiles, outAppFolder+filepath.Base(sourceFilePath))
}
// Analyse source files include directives and collect header names. Source files are first
// analysed by gcc to make sure to avoid directives that are commented or subjected to a macro
// and then "by hand" to sort the include directives of the gcc analysis (i.e., to avoid
// include directives that are not present in the source file currently analysed).
interdependMap := make(map[string][]string)
for _, outAppFile := range outAppFiles {
analysis := sourceFileIncludesAnalysis(outAppFile)
gccAnalysis, err := gccSourceFileIncludesAnalysis(outAppFile, outAppFolder)
if err != nil {
continue
}
interdependMap[filepath.Base(outAppFile)] = make([]string, 0)
for _, includeDirective := range gccAnalysis {
if u.Contains(analysis, includeDirective) {
interdependMap[filepath.Base(outAppFile)] =
append(interdependMap[filepath.Base(outAppFile)], includeDirective)
}
}
}
// Prune interdependence graph
extLibsList := requestUnikraftExtLibs()
extLibsList = append(extLibsList, "win32", "test", "TEST")
for _, extLib := range extLibsList {
pruneElemFiles(&interdependMap, extLib)
}
pruneRemovableFiles(&interdependMap)
// Remove pruned files from the out app folder
for _, outAppFile := range outAppFiles {
if _, ok := interdependMap[filepath.Base(outAppFile)]; !ok {
os.Remove(outAppFile)
}
}
// Create dot file
u.GenerateGraph(programName, outFolder+programName, interdependMap, nil)
return outAppFolder
}

176
srcs/dependtool/run_deptool.go
View file

@ -4,8 +4,6 @@ import (
"debug/elf"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
u "tools/srcs/common"
@ -13,110 +11,6 @@ import (
"github.com/fatih/color"
)
// sourceFileIncludesAnalysis collects all the include directives from a C/C++ source file.
//
// It returns a slice containing the found header names.
func sourceFileIncludesAnalysis(sourceFile string) []string {
var fileIncludes []string
fileLines, err := u.ReadLinesFile(sourceFile)
if err != nil {
u.PrintErr(err)
}
for _, line := range fileLines {
if strings.Contains(line, "#include") {
line = strings.ReplaceAll(line, " ", "")
line := strings.Split(line, "#include")[1]
if strings.HasPrefix(line, "\"") {
fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], "\"")+1])
} else if strings.HasPrefix(line, "<") {
fileIncludes = append(fileIncludes, line[1:strings.Index(line[1:], ">")+1])
}
}
}
return fileIncludes
}
// gccSourceFileIncludesAnalysis collects all the include directives from a C/C++ source file using
// the gcc preprocessor.
//
// It returns a slice containing the found header names.
func gccSourceFileIncludesAnalysis(sourceFile, tmpFolder string) []string {
var fileIncludes []string
outputStr, _ := ExecuteCommand("gcc", []string{"-E", sourceFile, "-I", tmpFolder})
outputSlice := strings.Split(outputStr, "\n")
for _, line := range outputSlice {
// Only interested in headers not coming from the standard library
if strings.Contains(line, "\""+tmpFolder) {
line = strings.Split(line, "\""+tmpFolder)[1]
includeDirective := line[0:strings.Index(line[0:], "\"")]
if !u.Contains(fileIncludes, includeDirective) {
fileIncludes = append(fileIncludes, includeDirective)
}
}
}
return fileIncludes
}
// runInterdependAnalyser collects all the included headers names (i.e., dependencies) from each
// C/C++ source file of a program and builds an interdependence graph (dot file) between all these
// source files.
func runInterdependAnalyser(programPath, programName, outFolder string) {
// Find all program source files
sourceFiles, err := findSourcesFiles(getProgramFolder(programPath))
if err != nil {
u.PrintErr(err)
}
// Create a temporary folder and copy all source files into it for use with the gcc
// preprocessor
tmpFolder := "tmp/"
_, err = u.CreateFolder(tmpFolder)
if err != nil {
u.PrintErr(err)
}
var tmpFiles []string
for _, sourceFilePath := range sourceFiles {
if err := u.CopyFileContents(sourceFilePath,
tmpFolder+filepath.Base(sourceFilePath)); err != nil {
u.PrintErr(err)
}
tmpFiles = append(tmpFiles, tmpFolder+filepath.Base(sourceFilePath))
}
// Analyse source files include directives and collect header names. Source files are first
// analysed "by hand" to get all their include directives and then by gcc to make sure to avoid
// directives that are commented or subjected to a macro.
interdependMap := make(map[string][]string)
for _, tmpFile := range tmpFiles {
interdependMap[filepath.Base(tmpFile)] = make([]string, 0)
analysis := sourceFileIncludesAnalysis(tmpFile)
gccAnalysis := gccSourceFileIncludesAnalysis(tmpFile, tmpFolder)
for _, includeDirective := range gccAnalysis {
if u.Contains(analysis, includeDirective) {
interdependMap[filepath.Base(tmpFile)] =
append(interdependMap[filepath.Base(tmpFile)], includeDirective)
}
}
}
// Create dot file
u.GenerateGraph(programName, outFolder+programName, interdependMap, nil)
// Remove tmp folder
u.PrintInfo("Remove folder " + tmpFolder)
_ = os.RemoveAll(tmpFolder)
}
// RunAnalyserTool allows to run the dependency analyser tool.
func RunAnalyserTool(homeDir string, data *u.Data) {
@ -136,10 +30,11 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
u.PrintErr(err)
}
// Get the kind of analysis (0: both; 1: static; 2: dynamic)
// Get the kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: sources; 5:
// stripped-down app and json for buildtool)
typeAnalysis := *args.IntArg[typeAnalysis]
if typeAnalysis < 0 || typeAnalysis > 2 {
u.PrintErr(errors.New("analysis argument must be between [0,2]"))
if typeAnalysis < 0 || typeAnalysis > 5 {
u.PrintErr(errors.New("analysis argument must be between [0,5]"))
}
// Get program path
@ -175,7 +70,8 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
// Run static analyser
if typeAnalysis == 0 || typeAnalysis == 1 {
u.PrintHeader1("(1.1) RUN STATIC ANALYSIS")
runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder, data)
runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder,
data)
}
// Run dynamic analyser
@ -189,6 +85,24 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
}
}
// Run interdependence analyser
if typeAnalysis == 0 || typeAnalysis == 3 {
u.PrintHeader1("(1.3) RUN INTERDEPENDENCE ANALYSIS")
_ = runInterdependAnalyser(programPath, programName, outFolder)
}
// Run sources analyser
if typeAnalysis == 0 || typeAnalysis == 4 {
u.PrintHeader1("(1.4) RUN SOURCES ANALYSIS")
runSourcesAnalyser(getProgramFolder(programPath), data)
}
// Prepare stripped-down app for buildtool
if typeAnalysis == 5 {
u.PrintHeader1("(1.5) PREPARE STRIPPED-DOWN APP AND JSON FOR BUILDTOOL")
runSourcesAnalyser(runInterdependAnalyser(programPath, programName, outFolder), data)
}
// Save Data to JSON
if err = u.RecordDataJson(outFolder+programName, data); err != nil {
u.PrintErr(err)
@ -201,11 +115,6 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
if *args.BoolArg[fullDepsArg] {
saveGraph(programName, outFolder, data)
}
// Create source files interdependence graph if interdependence option is set
if *args.BoolArg[interdependArg] {
runInterdependAnalyser(programPath, programName, outFolder)
}
}
// displayProgramDetails display various information such path, background, ...
@ -321,6 +230,12 @@ func runDynamicAnalyser(args *u.Arguments, programName, programPath,
}
}
// runDynamicAnalyser runs the sources analyser.
func runSourcesAnalyser(programPath string, data *u.Data) {
sourcesAnalyser(data, programPath)
}
// saveGraph saves dependency graphs of a given app into the output folder.
func saveGraph(programName, outFolder string, data *u.Data) {
@ -340,6 +255,12 @@ func saveGraph(programName, outFolder string, data *u.Data) {
}
}
// runInterdependAnalyser runs the interdependence analyser.
func runInterdependAnalyser(programPath, programName, outFolder string) string {
return interdependAnalyser(programPath, programName, outFolder)
}
/*
// /!\ MISSING "/" !!!
stringFile := "#include<stdlib.h>\n/* #include ta mère *\nint main() {\n\t// Salut bitch !\n\treturn 0;\n}"
@ -364,6 +285,7 @@ for i := 0; i < len(sliceFile); i++ {
}
}
// Remove dependencies whose files are not in program directory (e.g., stdio, stdlib, ...)
for internalFile, dependencies := range interdependMap {
var internalDep []string
@ -376,6 +298,7 @@ for i := 0; i < len(sliceFile); i++ {
interdependMap[internalFile] = internalDep
}
// Detect and print removable program source files (i.e., files that no other file depends
// on)
var removableFiles []string
@ -399,4 +322,27 @@ for i := 0; i < len(sliceFile); i++ {
}
fmt.Println("Removable program source files of ", programName, ":")
fmt.Println(removableFiles)
func isADependency(internalFile string, interdependMap* map[string][]string) bool {
for _, dependencies := range *interdependMap {
for _, dependency := range dependencies {
if internalFile == dependency {
return true
}
}
}
return false
}
mymap := make(map[string][]string)
mymap["file_win32"] = make([]string, 0)
mymap["file_1"] = []string{"file_win32"}
mymap["file_2"] = []string{"file_1"}
mymap["file_3"] = make([]string, 0)
mymap["file_4"] = []string{"file_3"}
mymap["file_openssl"] = []string{"file_3"}
mymap["file_5"] = []string{"file_openssl"}
mymap["file_6"] = []string{"file_5"}
*/

119
srcs/dependtool/sources_analyser.go Normal file
View file

@ -0,0 +1,119 @@
package dependtool
import (
"os"
"os/exec"
"path/filepath"
"strings"
u "tools/srcs/common"
)
// ---------------------------------Gather Data---------------------------------
// findSourcesFiles puts together all C/C++ source files found in a given application folder.
//
// It returns a slice containing the found source file names and an error if any, otherwise it
// returns nil.
func findSourcesFiles(workspace string) ([]string, error) {
var filenames []string
err := filepath.Walk(workspace,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
ext := filepath.Ext(info.Name())
if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
ext == ".hcc" {
filenames = append(filenames, path)
}
return nil
})
if err != nil {
return nil, err
}
return filenames, nil
}
// TODO REPLACE
// ExecuteCommand a single command without displaying the output.
//
// It returns a string which represents stdout and an error if any, otherwise
// it returns nil.
func ExecuteCommand(command string, arguments []string) (string, error) {
out, err := exec.Command(command, arguments...).CombinedOutput()
return string(out), err
}
// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in
// 'data'.
func addSourceFileSymbols(output string, data *u.SourcesData) {
outputTab := strings.Split(output, ",")
// Get the list of system calls
systemCalls := initSystemCalls()
for _, s := range outputTab {
if _, isSyscall := systemCalls[s]; isSyscall {
data.SystemCalls[s] = systemCalls[s]
} else {
data.Symbols[s] = ""
}
}
}
// extractPrototype executes the parserClang.py script on each source file to extracts all possible
// symbols of each of these files.
//
// It returns an error if any, otherwise it returns nil.
func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error {
for _, f := range sourcesFiltered {
script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool",
"parserClang.py")
output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f})
if err != nil {
u.PrintWarning("Incomplete analysis with file " + f)
continue
}
addSourceFileSymbols(output, data)
}
return nil
}
// gatherSourceFileSymbols gathers symbols of source files from a given application folder.
//
// It returns an error if any, otherwise it returns nil.
func gatherSourceFileSymbols(data *u.SourcesData, programPath string) error {
sourceFiles, err := findSourcesFiles(programPath)
if err != nil {
u.PrintErr(err)
}
if err := extractPrototype(sourceFiles, data); err != nil {
u.PrintErr(err)
}
return nil
}
// -------------------------------------Run-------------------------------------
// staticAnalyser runs the static analysis to get system calls and library calls of a given
// application.
func sourcesAnalyser(data *u.Data, programPath string) {
sourcesData := &data.SourcesData
// Init symbols members
sourcesData.Symbols = make(map[string]string)
sourcesData.SystemCalls = make(map[string]int)
// Detect symbols from source files
u.PrintHeader2("(*) Gathering symbols from source files")
if err := gatherSourceFileSymbols(sourcesData, programPath); err != nil {
u.PrintWarning(err)
}
}