Remove link processing temporarily & fix md links

src/scripts/transform_files.py

@@ -24,82 +24,7 @@ def remove_emojis(file_text):
     print(f"    Found: {found}")
     return re.sub(EMOJIS_regex, r"", file_text)
 
-
-def extract_markdown_links(md_file_content: str) -> List[MarkdownLink]:
-    """
-    Extracts markdown links from the given content.
-    """
-    markdown_links = []
-    link_pattern = r'\[([^\]]+)\]\(([^ ]+)( "([^"]+)")?\)\)?'
-
-    for match in re.finditer(link_pattern, md_file_content):
-        raw, text, url = match.group(0), match.group(1).strip('"').strip('**'), match.group(2)
-        title = match.group(4) if len(match.groups()) > 2 else None
-        title = title.replace("\\","") if title else None
-
-        if url.startswith("#"):
-            continue
-        if external:=url.startswith("http"):
-            raw_new = construct_external_link(raw, url, text, title)
-        else:
-            raw_new = construct_internal_link(raw, url, text, title)
-        markdown_links.append(MarkdownLink(raw, text, url, external, title, raw_new))
-
-    return markdown_links
-
-
-def construct_internal_link(raw, url, text, title):
-    """
-    Constructs a new internal link with the correct directory.
-    """
-    full_url = f"/MASTG/{get_directory_from_code(raw)}" if "0x" in raw else ""
-    title = f' "{title}"' if title else ""
-
-    return f"[{text}]({full_url}{title})".replace(".md", "")
-
-def construct_external_link(raw, url, text, title):
-    """
-    Constructs a new internal link with the correct directory.
-    """
-    text = title if title else text
-    title = f' "{title}"' if title else ""
-
-    return f"[{text}]({url}{title})"
-
-def get_directory_from_code(raw):
-    """
-    Maps directory code to directory name.
-    """
-    match = re.search(r'0x(\d{2})', raw)
-    if match:
-        directory = {
-            "01": "Intro",
-            "02": "Intro",
-            "03": "Intro",
-            "04": "General",
-            "05": "Android",
-            "06": "iOS",
-            "08": "Tools",
-            "09": "Intro",
-        }.get(match.group(1))
-        relative_path = raw[match.start():-1]
-        return f"{directory}/{relative_path}"
-    else:
-        return ""
-
-
-def get_links_from_anchor(links, anchor):
-    """
-    Extracts specific links from the anchor text.
-    """
-    return sorted(set([
-        re.search(r"\#([^ \")]*)(?=\s|\)|$)", link.raw).group(1)
-        for link in links
-        if not link.external and "#" in link.raw and anchor in link.raw
-    ]))
-
-
-def update_yaml_frontmatter(file_text, tools, examples, external_links, last_updated):
+def update_yaml_frontmatter(file_text, last_updated):
     """
     Updates the YAML frontmatter with the tools and examples list.
     """
@@ -111,13 +36,6 @@ def update_yaml_frontmatter(file_text, tools, examples, external_links, last_upd
         frontmatter_str = match.group(1)
         frontmatter = yaml.safe_load(frontmatter_str)
 
-        # Update the tools and examples in the frontmatter
-        frontmatter["tools"] = update_frontmatter_list(frontmatter.get("tools", []), tools)
-        frontmatter["examples"] = update_frontmatter_list(frontmatter.get("examples", []), examples)
-
-        # update with external links
-        frontmatter["resources"] = update_frontmatter_list(frontmatter.get("external_links", []), external_links)
-
         frontmatter["last_updated"] = last_updated
 
         # Replace the old frontmatter with the updated frontmatter
@@ -134,43 +52,6 @@ def update_frontmatter_list(current_list, new_items):
     updated_list = current_list + new_items
     return sorted(list(set(updated_list)))
 
-
-def links_to_markdown(links, title):
-    """
-    Converts a list of links to markdown.
-    """
-    section = ""
-    if len(links) > 0:
-        links = sorted(list(set([link.raw_new for link in links])))      
-        links_text = "\n".join([f"- {link}" for link in links])
-        # TODO links_text = f"\n\n### {title}\n\n{links_text}"
-        links_text = f"\n\n{links_text}"
-        section += links_text
-    return section
-
-
-def split_links(links):
-    internal_links = [link for link in links if link.external is False]
-    external_links = [link for link in links if link.external is True]
-
-    return internal_links, external_links
-
-def update_internal_links(file_text, links):
-    new_text = file_text
-    for link in links:        
-        new_text = new_text.replace(link.raw, link.raw_new)
-    return new_text
-
-def create_resources_section(internal_links, external_links):
-    # TODO internal_links_section = links_to_markdown(internal_links, "Internal")
-    external_links_section = links_to_markdown(external_links, "External")
-    # TODO resources_section = internal_links_section + external_links_section
-    resources_section = external_links_section
-
-    if resources_section != "":
-        resources_section = "\n\n## Resources" + resources_section + "\n"
-    return resources_section
-
 def process_markdown_files(folder):
     """
     Processes all markdown files in the given folder.
@@ -183,23 +64,11 @@ def process_markdown_files(folder):
                 if markdown_file.name == "index.md":
                     continue
                 file_content = markdown_file.read_text()
-                links = extract_markdown_links(file_content)
-
-                internal_links, external_links = split_links(links)
-
-                tools = get_links_from_anchor(internal_links, "0x08a")
-                examples = get_links_from_anchor(internal_links, "0x08b")
-
-                resources_section = create_resources_section(internal_links, external_links)
-
-                file_content = update_internal_links(file_content, internal_links)
-
-                external_links = [link.url for link in external_links]
 
                 last_updated = git_data.get_last_commit_date(Path(markdown_file.as_posix().replace('docs/MASTG', '.')).absolute().as_posix())
 
-                updated_content = update_yaml_frontmatter(file_content, tools, examples, external_links, last_updated)
-                markdown_file.write_text(updated_content + resources_section)
+                updated_content = update_yaml_frontmatter(file_content, last_updated)
+                markdown_file.write_text(updated_content)
 
 
 process_markdown_files("docs/MASTG")

techniques/android/MASTG-TECH-0017.md

@@ -19,7 +19,7 @@ Android decompilers go one step further and attempt to convert Android bytecode
 
 Alternatively you can use the [APKLab](0x08a-Testing-Tools.md#apklab) extension for Visual Studio Code or run [apkx](0x08a-Testing-Tools.md#apkx) on your APK or use the exported files from the previous tools to open the reversed source code on your preferred IDE.
 
-In the following example we'll be using [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1). First, let's install the app on a device or emulator and run it to see what the crackme is about.
+In the following example we'll be using [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md). First, let's install the app on a device or emulator and run it to see what the crackme is about.
 
 <img src="Images/Chapters/0x05c/crackme-1.png" width="400px" />
 

techniques/android/MASTG-TECH-0023.md

@@ -3,7 +3,7 @@ title: Reviewing Decompiled Java Code
 platform: android
 ---
 
-Following the example from ["Decompiling Java Code"](#decompiling-java-code), we assume that you've successfully decompiled and opened the [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) in IntelliJ. As soon as IntelliJ has indexed the code, you can browse it just like you'd browse any other Java project. Note that many of the decompiled packages, classes, and methods have weird one-letter names; this is because the bytecode has been "minified" with ProGuard at build time. This is a basic type of [obfuscation](0x04c-Tampering-and-Reverse-Engineering.md#obfuscation) that makes the bytecode a little more difficult to read, but with a fairly simple app like this one, it won't cause you much of a headache. When you're analyzing a more complex app, however, it can get quite annoying.
+Following the example from ["Decompiling Java Code"](#decompiling-java-code), we assume that you've successfully decompiled and opened the [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) in IntelliJ. As soon as IntelliJ has indexed the code, you can browse it just like you'd browse any other Java project. Note that many of the decompiled packages, classes, and methods have weird one-letter names; this is because the bytecode has been "minified" with ProGuard at build time. This is a basic type of [obfuscation](0x04c-Tampering-and-Reverse-Engineering.md#obfuscation) that makes the bytecode a little more difficult to read, but with a fairly simple app like this one, it won't cause you much of a headache. When you're analyzing a more complex app, however, it can get quite annoying.
 
 When analyzing obfuscated code, annotating class names, method names, and other identifiers as you go along is a good practice. Open the `MainActivity` class in the package `sg.vantagepoint.uncrackable1`. The method `verify` is called when you tap the "verify" button. This method passes the user input to a static method called `a.a`, which returns a boolean value. It seems plausible that `a.a` verifies user input, so we'll refactor the code to reflect this.
 
@@ -52,4 +52,4 @@ public class a {
 
 Now you're getting somewhere: it's simply standard AES-ECB. Looks like the Base64 string stored in `arrby1` in `check_input` is a ciphertext. It is decrypted with 128bit AES, then compared with the user input. As a bonus task, try to decrypt the extracted ciphertext and find the secret value!
 
-A faster way to get the decrypted string is to add dynamic analysis. We'll revisit [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) later to show how (e.g. in the Debugging section), so don't delete the project yet!
+A faster way to get the decrypted string is to add dynamic analysis. We'll revisit [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) later to show how (e.g. in the Debugging section), so don't delete the project yet!

techniques/android/MASTG-TECH-0026.md

@@ -9,7 +9,7 @@ Thanks to tools like [objection](0x08a-Testing-Tools.md#objection), you can patc
 
 Now you can use objection to dynamically analyze the application on non-rooted devices.
 
-The following commands summarize how to patch and start dynamic analysis using objection using the [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) as an example:
+The following commands summarize how to patch and start dynamic analysis using objection using the [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) as an example:
 
 ```bash
 # Download the Uncrackable APK

techniques/android/MASTG-TECH-0031.md

@@ -13,7 +13,7 @@ Dalvik and ART support the JDWP, a protocol for communication between the debugg
 
 A JDWP debugger allows you to step through Java code, set breakpoints on Java methods, and inspect and modify local and instance variables. You'll use a JDWP debugger most of the time you debug "normal" Android apps (i.e., apps that don't make many calls to native libraries).
 
-In the following section, we'll show how to solve the [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) with jdb alone. Note that this is not an _efficient_ way to solve this crackme. Actually you can do it much faster with Frida and other methods, which we'll introduce later in the guide. This, however, serves as an introduction to the capabilities of the Java debugger.
+In the following section, we'll show how to solve the [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) with jdb alone. Note that this is not an _efficient_ way to solve this crackme. Actually you can do it much faster with Frida and other methods, which we'll introduce later in the guide. This, however, serves as an introduction to the capabilities of the Java debugger.
 
 ## Debugging with jdb
 
@@ -44,7 +44,7 @@ You're now attached to the suspended process and ready to go ahead with the jdb
 - clear _method_: remove a method breakpoint
 - set _lvalue_ = _expr_:  assign new value to field/variable/array element
 
-Let's revisit the decompiled code from the [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) and think about possible solutions. A good approach would be suspending the app in a state where the secret string is held in a variable in plain text so you can retrieve it. Unfortunately, you won't get that far unless you deal with the root/tampering detection first.
+Let's revisit the decompiled code from the [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) and think about possible solutions. A good approach would be suspending the app in a state where the secret string is held in a variable in plain text so you can retrieve it. Unfortunately, you won't get that far unless you deal with the root/tampering detection first.
 
 Review the code and you'll see that the method `sg.vantagepoint.uncrackable1.MainActivity.a` displays the "This in unacceptable..." message box. This method creates an `AlertDialog` and sets a listener class for the `onClick` event. This class (named `b`) has a callback method will terminates the app once the user taps the **OK** button. To prevent the user from simply canceling the dialog, the `setCancelable` method is called.
 
@@ -128,7 +128,7 @@ This is the plaintext string you're looking for!
 
 Setting up a project in an IDE with the decompiled sources is a neat trick that allows you to set method breakpoints directly in the source code. In most cases, you should be able to single-step through the app and inspect the state of variables with the GUI. The experience won't be perfect, it's not the original source code after all, so you won't be able to set line breakpoints and things will sometimes simply not work correctly. Then again, reversing code is never easy, and efficiently navigating and debugging plain old Java code is a pretty convenient way of doing it. A similar method has been described in the [NetSPI blog](https://blog.netspi.com/attacking-android-applications-with-debuggers/ "NetSPI Blog - Attacking Android Applications with Debuggers").
 
-To set up IDE debugging, first create your Android project in IntelliJ and copy the decompiled Java sources into the source folder as described above in the "[Reviewing Decompiled Java Code](#reviewing-decompiled-java-code "Reviewing Decompiled Java Code")" section. On the device, choose the app as **debug app** on the "Developer options" ([UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) in this tutorial), and make sure you've switched on the "Wait For Debugger" feature.
+To set up IDE debugging, first create your Android project in IntelliJ and copy the decompiled Java sources into the source folder as described above in the "[Reviewing Decompiled Java Code](#reviewing-decompiled-java-code "Reviewing Decompiled Java Code")" section. On the device, choose the app as **debug app** on the "Developer options" ([UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) in this tutorial), and make sure you've switched on the "Wait For Debugger" feature.
 
 Once you tap the app icon from the launcher, it will be suspended in "Wait For Debugger" mode.
 

techniques/android/MASTG-TECH-0040.md

@@ -3,7 +3,7 @@ title: Waiting for the Debugger
 platform: android
 ---
 
-The [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) is not stupid: it notices that it has been run in debuggable mode and reacts by shutting down. A modal dialog is shown immediately, and the crackme terminates once you tap "OK".
+The [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) is not stupid: it notices that it has been run in debuggable mode and reacts by shutting down. A modal dialog is shown immediately, and the crackme terminates once you tap "OK".
 
 Fortunately, Android's "Developer options" contain the useful "Wait for Debugger" feature, which allows you to automatically suspend an app during startup until a JDWP debugger connects. With this feature, you can connect the debugger before the detection mechanism runs, and trace, debug, and deactivate that mechanism. It's really an unfair advantage, but, on the other hand, reverse engineers never play fair!
 

techniques/android/MASTG-TECH-0043.md

@@ -66,7 +66,7 @@ Just like regular Android apps, modules for Xposed are developed and deployed wi
 
 ## Frida
 
-We'll use Frida to solve the [UnCrackable App for Android Level 1](0x08b-Reference-Apps.md#android-uncrackable-l1) and demonstrate how we can easily bypass root detection and extract secret data from the app.
+We'll use Frida to solve the [UnCrackable App for Android Level 1](../../apps/android/MASTG-APP-0003.md) and demonstrate how we can easily bypass root detection and extract secret data from the app.
 
 When you start the crackme app on an emulator or a rooted device, you'll find that the it presents a dialog box and exits as soon as you press "OK" because it detected root:
 

techniques/android/MASTG-TECH-0100.md

@@ -3,7 +3,7 @@ title: Logging Sensitive Data from Network Traffic
 platform: android
 ---
 
-[mitmproxy](MASTG-TOOL-0097) can be used to intercept network traffic from Android apps. This technique is useful for identifying sensitive data that is sent over the network, as well as for identifying potential security vulnerabilities.
+[mitmproxy](../../tools/network/MASTG-TOOL-0097.md) can be used to intercept network traffic from Android apps. This technique is useful for identifying sensitive data that is sent over the network, as well as for identifying potential security vulnerabilities.
 
 Once with mitmproxy installed and your device configured to use it, you can create a python script to filter the traffic and extract the sensitive data. For example, the following script will extract all the data sent in the requests and responses only if the data is considered sensitive. For this example we consider sensitive data to be any data that contains the strings "dummyPassword" or "sampleUser", so we include them in the `SENSITIVE_STRINGS` list.